* [PATCH 2/2] crypto: arm/crc32 - accelerated support based on x86 SSE implementation
From: Ard Biesheuvel @ 2016-11-26 20:15 UTC (permalink / raw)
To: linux-crypto, herbert, linux-arm-kernel, catalin.marinas,
will.deacon, linux
Cc: steve.capper, Ard Biesheuvel
In-Reply-To: <1480191314-2331-1-git-send-email-ard.biesheuvel@linaro.org>
This is a combination of the the Intel algorithm implemented using SSE
and PCLMULQDQ instructions from arch/x86/crypto/crc32-pclmul_asm.S, and
the new CRC32 extensions introduced for both 32-bit and 64-bit ARM in
version 8 of the architecture.
The PMULL/NEON algorithm is faster, but operates on blocks of at least
64 bytes, and on multiples of 16 bytes only. For the remaining input,
or for all input on systems that lack the PMULL 64x64->128 instructions,
the CRC32 instructions will be used.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm/crypto/Kconfig | 5 +
arch/arm/crypto/Makefile | 2 +
arch/arm/crypto/crc32-ce-core.S | 257 ++++++++++++++++++++
arch/arm/crypto/crc32-ce-glue.c | 129 ++++++++++
4 files changed, 393 insertions(+)
diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig
index fce801fa52a1..be5cb5a7d3fa 100644
--- a/arch/arm/crypto/Kconfig
+++ b/arch/arm/crypto/Kconfig
@@ -125,4 +125,9 @@ config CRYPTO_CRCT10DIF_ARM_CE
depends on KERNEL_MODE_NEON && CRC_T10DIF
select CRYPTO_HASH
+config CRYPTO_CRC32_ARM_CE
+ tristate "CRC32 digest algorithm using CRC and/or PMULL instructions"
+ depends on KERNEL_MODE_NEON && CRC32
+ select CRYPTO_HASH
+
endif
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
index fc77265014b7..b578a1820ab1 100644
--- a/arch/arm/crypto/Makefile
+++ b/arch/arm/crypto/Makefile
@@ -14,6 +14,7 @@ ce-obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o
ce-obj-$(CONFIG_CRYPTO_SHA2_ARM_CE) += sha2-arm-ce.o
ce-obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o
ce-obj-$(CONFIG_CRYPTO_CRCT10DIF_ARM_CE) += crct10dif-arm-ce.o
+ce-obj-$(CONFIG_CRYPTO_CRC32_ARM_CE) += crc32-arm-ce.o
ifneq ($(ce-obj-y)$(ce-obj-m),)
ifeq ($(call as-instr,.fpu crypto-neon-fp-armv8,y,n),y)
@@ -38,6 +39,7 @@ sha2-arm-ce-y := sha2-ce-core.o sha2-ce-glue.o
aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o
ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o
crct10dif-arm-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o
+crc32-arm-ce-y:= crc32-ce-core.o crc32-ce-glue.o
quiet_cmd_perl = PERL $@
cmd_perl = $(PERL) $(<) > $(@)
diff --git a/arch/arm/crypto/crc32-ce-core.S b/arch/arm/crypto/crc32-ce-core.S
new file mode 100644
index 000000000000..ef671f040672
--- /dev/null
+++ b/arch/arm/crypto/crc32-ce-core.S
@@ -0,0 +1,257 @@
+/*
+ * Accelerated CRC32 using ARM CRC, NEON and Crypto Extensions instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/* GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see http://www.gnu.org/licenses
+ *
+ * Please visit http://www.xyratex.com/contact if you need additional
+ * information or have any questions.
+ *
+ * GPL HEADER END
+ */
+
+/*
+ * Copyright 2012 Xyratex Technology Limited
+ *
+ * Using hardware provided PCLMULQDQ instruction to accelerate the CRC32
+ * calculation.
+ * CRC32 polynomial:0x04c11db7(BE)/0xEDB88320(LE)
+ * PCLMULQDQ is a new instruction in Intel SSE4.2, the reference can be found
+ * at:
+ * http://www.intel.com/products/processor/manuals/
+ * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
+ * Volume 2B: Instruction Set Reference, N-Z
+ *
+ * Authors: Gregory Prestas <Gregory_Prestas@us.xyratex.com>
+ * Alexander Boyko <Alexander_Boyko@xyratex.com>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+ .text
+ .align 4
+ .arch armv8-a
+ .arch_extension crc
+ .fpu crypto-neon-fp-armv8
+
+ /*
+ * [x4*128+32 mod P(x) << 32)]' << 1 = 0x154442bd4
+ * #define CONSTANT_R1 0x154442bd4LL
+ *
+ * [(x4*128-32 mod P(x) << 32)]' << 1 = 0x1c6e41596
+ * #define CONSTANT_R2 0x1c6e41596LL
+ */
+.Lconstant_R2R1:
+ .quad 0x0000000154442bd4
+ .quad 0x00000001c6e41596
+
+ /*
+ * [(x128+32 mod P(x) << 32)]' << 1 = 0x1751997d0
+ * #define CONSTANT_R3 0x1751997d0LL
+ *
+ * [(x128-32 mod P(x) << 32)]' << 1 = 0x0ccaa009e
+ * #define CONSTANT_R4 0x0ccaa009eLL
+ */
+.Lconstant_R4R3:
+ .quad 0x00000001751997d0
+ .quad 0x00000000ccaa009e
+
+ /*
+ * [(x64 mod P(x) << 32)]' << 1 = 0x163cd6124
+ * #define CONSTANT_R5 0x163cd6124LL
+ */
+.Lconstant_R5:
+ .quad 0x0000000163cd6124
+
+.Lconstant_mask32:
+ .quad 0x00000000FFFFFFFF
+
+ /*
+ * #define CRCPOLY_TRUE_LE_FULL 0x1DB710641LL
+ *
+ * Barrett Reduction constant (u64`) = u` = (x**64 / P(x))`
+ * = 0x1F7011641LL
+ * #define CONSTANT_RU 0x1F7011641LL
+ */
+.Lconstant_RUpoly:
+ .quad 0x00000001DB710641
+ .quad 0x00000001F7011641
+
+ dCONSTANTl .req d0
+ dCONSTANTh .req d1
+ qCONSTANT .req q0
+
+ BUF .req r0
+ LEN .req r1
+ CRC .req r2
+
+ qzr .req q9
+
+ /**
+ * Calculate crc32
+ * BUF - buffer
+ * LEN - sizeof buffer (multiple of 16 bytes), LEN should be > 63
+ * CRC - initial crc32
+ * return %eax crc32
+ * uint crc32_pmull_le(unsigned char const *buffer,
+ * size_t len, uint crc32)
+ */
+ENTRY(crc32_pmull_le)
+ bic LEN, LEN, #15
+ vld1.8 {q1-q2}, [BUF]!
+ vld1.8 {q3-q4}, [BUF]!
+ vmov.i8 qzr, #0
+ vmov.i8 qCONSTANT, #0
+ vmov dCONSTANTl[0], CRC
+ veor.8 d2, d2, dCONSTANTl
+ sub LEN, LEN, #0x40
+ cmp LEN, #0x40
+ blt less_64
+
+ vldr dCONSTANTl, .Lconstant_R2R1
+ vldr dCONSTANTh, .Lconstant_R2R1 + 8
+
+loop_64: /* 64 bytes Full cache line folding */
+ sub LEN, LEN, #0x40
+
+ vmull.p64 q5, d3, dCONSTANTh
+ vmull.p64 q6, d5, dCONSTANTh
+ vmull.p64 q7, d7, dCONSTANTh
+ vmull.p64 q8, d9, dCONSTANTh
+
+ vmull.p64 q1, d2, dCONSTANTl
+ vmull.p64 q2, d4, dCONSTANTl
+ vmull.p64 q3, d6, dCONSTANTl
+ vmull.p64 q4, d8, dCONSTANTl
+
+ veor.8 q1, q1, q5
+ vld1.8 {q5}, [BUF]!
+ veor.8 q2, q2, q6
+ vld1.8 {q6}, [BUF]!
+ veor.8 q3, q3, q7
+ vld1.8 {q7}, [BUF]!
+ veor.8 q4, q4, q8
+ vld1.8 {q8}, [BUF]!
+
+ veor.8 q1, q1, q5
+ veor.8 q2, q2, q6
+ veor.8 q3, q3, q7
+ veor.8 q4, q4, q8
+
+ cmp LEN, #0x40
+ bge loop_64
+
+less_64: /* Folding cache line into 128bit */
+ vldr dCONSTANTl, .Lconstant_R4R3
+ vldr dCONSTANTh, .Lconstant_R4R3 + 8
+
+ vmull.p64 q5, d3, dCONSTANTh
+ vmull.p64 q1, d2, dCONSTANTl
+ veor.8 q1, q1, q5
+ veor.8 q1, q1, q2
+
+ vmull.p64 q5, d3, dCONSTANTh
+ vmull.p64 q1, d2, dCONSTANTl
+ veor.8 q1, q1, q5
+ veor.8 q1, q1, q3
+
+ vmull.p64 q5, d3, dCONSTANTh
+ vmull.p64 q1, d2, dCONSTANTl
+ veor.8 q1, q1, q5
+ veor.8 q1, q1, q4
+
+ teq LEN, #0
+ beq fold_64
+
+loop_16: /* Folding rest buffer into 128bit */
+ subs LEN, LEN, #0x10
+
+ vld1.8 {q2}, [BUF]!
+ vmull.p64 q5, d3, dCONSTANTh
+ vmull.p64 q1, d2, dCONSTANTl
+ veor.8 q1, q1, q5
+ veor.8 q1, q1, q2
+
+ bne loop_16
+
+fold_64:
+ /* perform the last 64 bit fold, also adds 32 zeroes
+ * to the input stream */
+ vmull.p64 q2, d2, dCONSTANTh
+ vext.8 q1, q1, qzr, #8
+ veor.8 q1, q1, q2
+
+ /* final 32-bit fold */
+ vldr dCONSTANTl, .Lconstant_R5
+ vldr d6, .Lconstant_mask32
+ vmov.i8 d7, #0
+
+ vext.8 q2, q1, qzr, #4
+ vand.8 d2, d2, d6
+ vmull.p64 q1, d2, dCONSTANTl
+ veor.8 q1, q1, q2
+
+ /* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
+ vldr dCONSTANTl, .Lconstant_RUpoly
+ vldr dCONSTANTh, .Lconstant_RUpoly + 8
+
+ vand.8 q2, q1, q3
+ vext.8 q2, qzr, q2, #8
+ vmull.p64 q2, d5, dCONSTANTh
+ vand.8 q2, q2, q3
+ vmull.p64 q2, d4, dCONSTANTl
+ veor.8 q1, q1, q2
+ vmov r0, s5
+
+ bx lr
+ENDPROC(crc32_pmull_le)
+
+ENTRY(crc32_armv8_le)
+ mov ip, r2
+0: subs ip, ip, #8
+ bmi 4f
+ ldrd r2, r3, [r1], #8
+ARM_BE8(rev r2, r2 )
+ARM_BE8(rev r3, r3 )
+ crc32w r0, r0, r2
+ crc32w r0, r0, r3
+ b 0b
+
+4: tst ip, #4
+ beq 2f
+ ldr r3, [r1], #4
+ARM_BE8(rev r3, r3 )
+ crc32w r0, r0, r3
+2: tst ip, #2
+ beq 1f
+ ldrh r3, [r1], #2
+ARM_BE8(rev16 r3, r3 )
+ crc32h r0, r0, r3
+1: tst ip, #1
+ bxeq lr
+ ldrb r3, [r1]
+ crc32b r0, r0, r3
+ bx lr
+ENDPROC(crc32_armv8_le)
diff --git a/arch/arm/crypto/crc32-ce-glue.c b/arch/arm/crypto/crc32-ce-glue.c
new file mode 100644
index 000000000000..61916b6b40e9
--- /dev/null
+++ b/arch/arm/crypto/crc32-ce-glue.c
@@ -0,0 +1,129 @@
+/*
+ * Accelerated CRC32 using ARM CRC, NEON and Crypto Extensions instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/crc32.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/string.h>
+
+#include <crypto/internal/hash.h>
+
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+#include <asm/simd.h>
+#include <asm/unaligned.h>
+
+#define PMULL_MIN_LEN 64L /* minimum size of buffer
+ * for crc32_pmull_le_16 */
+#define SCALE_F 16L /* size of NEON register */
+
+asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc);
+asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);
+
+static int crc32_pmull_cra_init(struct crypto_tfm *tfm)
+{
+ u32 *key = crypto_tfm_ctx(tfm);
+
+ *key = 0;
+ return 0;
+}
+
+static int crc32_pmull_setkey(struct crypto_shash *hash, const u8 *key,
+ unsigned int keylen)
+{
+ u32 *mctx = crypto_shash_ctx(hash);
+
+ if (keylen != sizeof(u32)) {
+ crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+ *mctx = le32_to_cpup((__le32 *)key);
+ return 0;
+}
+
+static int crc32_pmull_init(struct shash_desc *desc)
+{
+ u32 *mctx = crypto_shash_ctx(desc->tfm);
+ u32 *crc = shash_desc_ctx(desc);
+
+ *crc = *mctx;
+ return 0;
+}
+
+static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
+ unsigned int length)
+{
+ u32 *crc = shash_desc_ctx(desc);
+
+ if (length >= PMULL_MIN_LEN && may_use_simd() &&
+ (elf_hwcap2 & HWCAP2_PMULL)) {
+ kernel_neon_begin();
+ *crc = crc32_pmull_le(data, round_down(length, SCALE_F), *crc);
+ kernel_neon_end();
+
+ data += round_down(length, SCALE_F);
+ length %= SCALE_F;
+ }
+
+ if (length > 0) {
+ if (elf_hwcap2 & HWCAP2_CRC32)
+ *crc = crc32_armv8_le(*crc, data, length);
+ else
+ *crc = crc32_le(*crc, data, length);
+ }
+
+ return 0;
+}
+
+static int crc32_pmull_final(struct shash_desc *desc, u8 *out)
+{
+ u32 *crc = shash_desc_ctx(desc);
+
+ put_unaligned_le32(*crc, out);
+ return 0;
+}
+
+static struct shash_alg crc32_pmull_alg = {
+ .setkey = crc32_pmull_setkey,
+ .init = crc32_pmull_init,
+ .update = crc32_pmull_update,
+ .final = crc32_pmull_final,
+ .descsize = sizeof(u32),
+ .digestsize = sizeof(u32),
+
+ .base.cra_ctxsize = sizeof(u32),
+ .base.cra_init = crc32_pmull_cra_init,
+ .base.cra_name = "crc32",
+ .base.cra_driver_name = "crc32-arm-ce",
+ .base.cra_priority = 200,
+ .base.cra_blocksize = 1,
+ .base.cra_module = THIS_MODULE,
+};
+
+static int __init crc32_pmull_mod_init(void)
+{
+ if (!(elf_hwcap2 & (HWCAP2_PMULL|HWCAP2_CRC32)))
+ return -ENODEV;
+
+ return crypto_register_shash(&crc32_pmull_alg);
+}
+
+static void __exit crc32_pmull_mod_exit(void)
+{
+ crypto_unregister_shash(&crc32_pmull_alg);
+}
+
+module_init(crc32_pmull_mod_init);
+module_exit(crc32_pmull_mod_exit);
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("crc32");
--
2.7.4
^ permalink raw reply related
* [PATCH 1/2] crypto: arm64/crc32 - accelerated support based on x86 SSE implementation
From: Ard Biesheuvel @ 2016-11-26 20:15 UTC (permalink / raw)
To: linux-crypto, herbert, linux-arm-kernel, catalin.marinas,
will.deacon, linux
Cc: steve.capper, Ard Biesheuvel
In-Reply-To: <1480191314-2331-1-git-send-email-ard.biesheuvel@linaro.org>
This is a combination of the the Intel algorithm implemented using SSE
and PCLMULQDQ instructions from arch/x86/crypto/crc32-pclmul_asm.S, and
the new CRC32 extensions introduced for both 32-bit and 64-bit ARM in
version 8 of the architecture.
The PMULL/NEON algorithm is faster, but operates on blocks of at least
64 bytes, and on multiples of 16 bytes only. For the remaining input,
or for all input on systems that lack the PMULL 64x64->128 instructions,
the CRC32 instructions will be used.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/Kconfig | 6 +
arch/arm64/crypto/Makefile | 3 +
arch/arm64/crypto/crc32-ce-core.S | 246 ++++++++++++++++++++
arch/arm64/crypto/crc32-ce-glue.c | 124 ++++++++++
4 files changed, 379 insertions(+)
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 1b50671ffec3..11dc2ac1f2e5 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -58,4 +58,10 @@ config CRYPTO_CRC32_ARM64
tristate "CRC32 and CRC32C using optional ARMv8 instructions"
depends on ARM64
select CRYPTO_HASH
+
+config CRYPTO_CRC32_ARM64_CE
+ tristate "CRC32 digest algorithm using PMULL instructions"
+ depends on ARM64 && KERNEL_MODE_NEON
+ select CRYPTO_HASH
+
endif
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index 36fd3eb4201b..144387805a46 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -20,6 +20,9 @@ ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o
obj-$(CONFIG_CRYPTO_CRCT10DIF_ARM64_CE) += crct10dif-ce.o
crct10dif-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o
+obj-$(CONFIG_CRYPTO_CRC32_ARM64_CE) += crc32-ce.o
+crc32-ce-y:= crc32-ce-core.o crc32-ce-glue.o
+
obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o
CFLAGS_aes-ce-cipher.o += -march=armv8-a+crypto
diff --git a/arch/arm64/crypto/crc32-ce-core.S b/arch/arm64/crypto/crc32-ce-core.S
new file mode 100644
index 000000000000..eff7fe100dab
--- /dev/null
+++ b/arch/arm64/crypto/crc32-ce-core.S
@@ -0,0 +1,246 @@
+/*
+ * Accelerated CRC32 using arm64 CRC, NEON and Crypto Extensions instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/* GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see http://www.gnu.org/licenses
+ *
+ * Please visit http://www.xyratex.com/contact if you need additional
+ * information or have any questions.
+ *
+ * GPL HEADER END
+ */
+
+/*
+ * Copyright 2012 Xyratex Technology Limited
+ *
+ * Using hardware provided PCLMULQDQ instruction to accelerate the CRC32
+ * calculation.
+ * CRC32 polynomial:0x04c11db7(BE)/0xEDB88320(LE)
+ * PCLMULQDQ is a new instruction in Intel SSE4.2, the reference can be found
+ * at:
+ * http://www.intel.com/products/processor/manuals/
+ * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
+ * Volume 2B: Instruction Set Reference, N-Z
+ *
+ * Authors: Gregory Prestas <Gregory_Prestas@us.xyratex.com>
+ * Alexander Boyko <Alexander_Boyko@xyratex.com>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+
+ .text
+ .align 4
+ .cpu generic+crypto+crc
+
+ /*
+ * [x4*128+32 mod P(x) << 32)]' << 1 = 0x154442bd4
+ * #define CONSTANT_R1 0x154442bd4LL
+ *
+ * [(x4*128-32 mod P(x) << 32)]' << 1 = 0x1c6e41596
+ * #define CONSTANT_R2 0x1c6e41596LL
+ */
+.Lconstant_R2R1:
+ .octa 0x00000001c6e415960000000154442bd4
+
+ /*
+ * [(x128+32 mod P(x) << 32)]' << 1 = 0x1751997d0
+ * #define CONSTANT_R3 0x1751997d0LL
+ *
+ * [(x128-32 mod P(x) << 32)]' << 1 = 0x0ccaa009e
+ * #define CONSTANT_R4 0x0ccaa009eLL
+ */
+.Lconstant_R4R3:
+ .octa 0x00000000ccaa009e00000001751997d0
+
+ /*
+ * [(x64 mod P(x) << 32)]' << 1 = 0x163cd6124
+ * #define CONSTANT_R5 0x163cd6124LL
+ */
+.Lconstant_R5:
+ .octa 0x00000000000000000000000163cd6124
+.Lconstant_mask32:
+ .octa 0x000000000000000000000000FFFFFFFF
+
+ /*
+ * #define CRCPOLY_TRUE_LE_FULL 0x1DB710641LL
+ *
+ * Barrett Reduction constant (u64`) = u` = (x**64 / P(x))`
+ * = 0x1F7011641LL
+ * #define CONSTANT_RU 0x1F7011641LL
+ */
+.Lconstant_RUpoly:
+ .octa 0x00000001F701164100000001DB710641
+
+ vCONSTANT .req v0
+ dCONSTANT .req d0
+ qCONSTANT .req q0
+
+ BUF .req x0
+ LEN .req x1
+ CRC .req x2
+
+ vzr .req v9
+
+ /**
+ * Calculate crc32
+ * BUF - buffer
+ * LEN - sizeof buffer (multiple of 16 bytes), LEN should be > 63
+ * CRC - initial crc32
+ * return %eax crc32
+ * uint crc32_pmull_le(unsigned char const *buffer,
+ * size_t len, uint crc32)
+ */
+ENTRY(crc32_pmull_le)
+ bic LEN, LEN, #15
+ ld1 {v1.16b-v4.16b}, [BUF], #0x40
+ movi vzr.16b, #0
+ fmov dCONSTANT, CRC
+ eor v1.16b, v1.16b, vCONSTANT.16b
+ sub LEN, LEN, #0x40
+ cmp LEN, #0x40
+ b.lt less_64
+
+ ldr qCONSTANT, .Lconstant_R2R1
+
+loop_64: /* 64 bytes Full cache line folding */
+ sub LEN, LEN, #0x40
+
+ pmull2 v5.1q, v1.2d, vCONSTANT.2d
+ pmull2 v6.1q, v2.2d, vCONSTANT.2d
+ pmull2 v7.1q, v3.2d, vCONSTANT.2d
+ pmull2 v8.1q, v4.2d, vCONSTANT.2d
+
+ pmull v1.1q, v1.1d, vCONSTANT.1d
+ pmull v2.1q, v2.1d, vCONSTANT.1d
+ pmull v3.1q, v3.1d, vCONSTANT.1d
+ pmull v4.1q, v4.1d, vCONSTANT.1d
+
+ eor v1.16b, v1.16b, v5.16b
+ ld1 {v5.16b}, [BUF], #0x10
+ eor v2.16b, v2.16b, v6.16b
+ ld1 {v6.16b}, [BUF], #0x10
+ eor v3.16b, v3.16b, v7.16b
+ ld1 {v7.16b}, [BUF], #0x10
+ eor v4.16b, v4.16b, v8.16b
+ ld1 {v8.16b}, [BUF], #0x10
+
+ eor v1.16b, v1.16b, v5.16b
+ eor v2.16b, v2.16b, v6.16b
+ eor v3.16b, v3.16b, v7.16b
+ eor v4.16b, v4.16b, v8.16b
+
+ cmp LEN, #0x40
+ b.ge loop_64
+
+less_64: /* Folding cache line into 128bit */
+ ldr qCONSTANT, .Lconstant_R4R3
+
+ pmull2 v5.1q, v1.2d, vCONSTANT.2d
+ pmull v1.1q, v1.1d, vCONSTANT.1d
+ eor v1.16b, v1.16b, v5.16b
+ eor v1.16b, v1.16b, v2.16b
+
+ pmull2 v5.1q, v1.2d, vCONSTANT.2d
+ pmull v1.1q, v1.1d, vCONSTANT.1d
+ eor v1.16b, v1.16b, v5.16b
+ eor v1.16b, v1.16b, v3.16b
+
+ pmull2 v5.1q, v1.2d, vCONSTANT.2d
+ pmull v1.1q, v1.1d, vCONSTANT.1d
+ eor v1.16b, v1.16b, v5.16b
+ eor v1.16b, v1.16b, v4.16b
+
+ cbz LEN, fold_64
+
+loop_16: /* Folding rest buffer into 128bit */
+ subs LEN, LEN, #0x10
+
+ ld1 {v2.16b}, [BUF], #0x10
+ pmull2 v5.1q, v1.2d, vCONSTANT.2d
+ pmull v1.1q, v1.1d, vCONSTANT.1d
+ eor v1.16b, v1.16b, v5.16b
+ eor v1.16b, v1.16b, v2.16b
+
+ b.ne loop_16
+
+fold_64:
+ /* perform the last 64 bit fold, also adds 32 zeroes
+ * to the input stream */
+ ext v2.16b, v1.16b, v1.16b, #8
+ pmull2 v2.1q, v2.2d, vCONSTANT.2d
+ ext v1.16b, v1.16b, vzr.16b, #8
+ eor v1.16b, v1.16b, v2.16b
+
+ /* final 32-bit fold */
+ ldr qCONSTANT, .Lconstant_R5
+ ldr q3, .Lconstant_mask32
+
+ ext v2.16b, v1.16b, vzr.16b, #4
+ and v1.16b, v1.16b, v3.16b
+ pmull v1.1q, v1.1d, vCONSTANT.1d
+ eor v1.16b, v1.16b, v2.16b
+
+ /* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
+ ldr qCONSTANT, .Lconstant_RUpoly
+
+ and v2.16b, v1.16b, v3.16b
+ ext v2.16b, vzr.16b, v2.16b, #8
+ pmull2 v2.1q, v2.2d, vCONSTANT.2d
+ and v2.16b, v2.16b, v3.16b
+ pmull v2.1q, v2.1d, vCONSTANT.1d
+ eor v1.16b, v1.16b, v2.16b
+ mov w0, v1.s[1]
+
+ ret
+ENDPROC(crc32_pmull_le)
+
+ENTRY(crc32_armv8_le)
+0: subs x2, x2, #16
+ b.mi 8f
+ ldp x3, x4, [x1], #16
+CPU_BE( rev x3, x3 )
+CPU_BE( rev x4, x4 )
+ crc32x w0, w0, x3
+ crc32x w0, w0, x4
+ b 0b
+
+8: tbz x2, #3, 4f
+ ldr x3, [x1], #8
+CPU_BE( rev x3, x3 )
+ crc32x w0, w0, x3
+4: tbz x2, #2, 2f
+ ldr w3, [x1], #4
+CPU_BE( rev w3, w3 )
+ crc32w w0, w0, w3
+2: tbz x2, #1, 1f
+ ldrh w3, [x1], #2
+CPU_BE( rev16 w3, w3 )
+ crc32h w0, w0, w3
+1: tbz x2, #0, 0f
+ ldrb w3, [x1]
+ crc32b w0, w0, w3
+0: ret
+ENDPROC(crc32_armv8_le)
diff --git a/arch/arm64/crypto/crc32-ce-glue.c b/arch/arm64/crypto/crc32-ce-glue.c
new file mode 100644
index 000000000000..567203f29ac6
--- /dev/null
+++ b/arch/arm64/crypto/crc32-ce-glue.c
@@ -0,0 +1,124 @@
+/*
+ * Accelerated CRC32 using arm64 NEON and Crypto Extensions instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/cpufeature.h>
+#include <linux/crc32.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/string.h>
+
+#include <crypto/internal/hash.h>
+
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+#include <asm/unaligned.h>
+
+#define PMULL_MIN_LEN 64L /* minimum size of buffer
+ * for crc32_pmull_le_16 */
+#define SCALE_F 16L /* size of NEON register */
+
+asmlinkage u32 crc32_pmull_le(const u8 buf[], u64 len, u32 init_crc);
+asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u64 len);
+
+static int crc32_pmull_cra_init(struct crypto_tfm *tfm)
+{
+ u32 *key = crypto_tfm_ctx(tfm);
+
+ *key = 0;
+ return 0;
+}
+
+static int crc32_pmull_setkey(struct crypto_shash *hash, const u8 *key,
+ unsigned int keylen)
+{
+ u32 *mctx = crypto_shash_ctx(hash);
+
+ if (keylen != sizeof(u32)) {
+ crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+ *mctx = le32_to_cpup((__le32 *)key);
+ return 0;
+}
+
+static int crc32_pmull_init(struct shash_desc *desc)
+{
+ u32 *mctx = crypto_shash_ctx(desc->tfm);
+ u32 *crc = shash_desc_ctx(desc);
+
+ *crc = *mctx;
+ return 0;
+}
+
+static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
+ unsigned int length)
+{
+ u32 *crc = shash_desc_ctx(desc);
+
+ if (length >= PMULL_MIN_LEN) {
+ kernel_neon_begin_partial(10);
+ *crc = crc32_pmull_le(data, round_down(length, SCALE_F), *crc);
+ kernel_neon_end();
+
+ data += round_down(length, SCALE_F);
+ length %= SCALE_F;
+ }
+
+ if (length > 0) {
+ if (elf_hwcap & HWCAP_CRC32)
+ *crc = crc32_armv8_le(*crc, data, length);
+ else
+ *crc = crc32_le(*crc, data, length);
+ }
+
+ return 0;
+}
+
+static int crc32_pmull_final(struct shash_desc *desc, u8 *out)
+{
+ u32 *crc = shash_desc_ctx(desc);
+
+ put_unaligned_le32(*crc, out);
+ return 0;
+}
+
+static struct shash_alg crc32_pmull_alg = {
+ .setkey = crc32_pmull_setkey,
+ .init = crc32_pmull_init,
+ .update = crc32_pmull_update,
+ .final = crc32_pmull_final,
+ .descsize = sizeof(u32),
+ .digestsize = sizeof(u32),
+
+ .base.cra_ctxsize = sizeof(u32),
+ .base.cra_init = crc32_pmull_cra_init,
+ .base.cra_name = "crc32",
+ .base.cra_driver_name = "crc32-arm64-ce",
+ .base.cra_priority = 200,
+ .base.cra_blocksize = 1,
+ .base.cra_module = THIS_MODULE,
+};
+
+static int __init crc32_pmull_mod_init(void)
+{
+ return crypto_register_shash(&crc32_pmull_alg);
+}
+
+static void __exit crc32_pmull_mod_exit(void)
+{
+ crypto_unregister_shash(&crc32_pmull_alg);
+}
+
+module_cpu_feature_match(PMULL, crc32_pmull_mod_init);
+module_exit(crc32_pmull_mod_exit);
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
--
2.7.4
^ permalink raw reply related
* [PATCH 0/2] CRC32 for ARM and arm64 using PMULL and CRC instructions
From: Ard Biesheuvel @ 2016-11-26 20:15 UTC (permalink / raw)
To: linux-crypto, herbert, linux-arm-kernel, catalin.marinas,
will.deacon, linux
Cc: steve.capper, Ard Biesheuvel
Version 8 of the ARM architecture introduces both a set of dedicated CRC32
instructions, and a 64x64 to 128 bit polynomial multiplication instruction,
both of which can be used to accelerate CRC32 calculations.
These patches contains ports of the existing polynomial multiplication based
CRC32 code that resides in arch/x86/crypto/crc32-pclmul_asm.S, but since that
algorithm operates on multiples of 16 bytes only, and requires at least 64
bytes of input, the remainders are calculated with the CRC32 instructions,
if available.
These patches apply on top of the CRC-T10DIF series I sent out last Thursday.
https://git.kernel.org/cgit/linux/kernel/git/ardb/linux.git/log/?h=crc32
Ard Biesheuvel (2):
crypto: arm64/crc32 - accelerated support based on x86 SSE
implementation
crypto: arm/crc32 - accelerated support based on x86 SSE
implementation
arch/arm/crypto/Kconfig | 5 +
arch/arm/crypto/Makefile | 2 +
arch/arm/crypto/crc32-ce-core.S | 257 ++++++++++++++++++++
arch/arm/crypto/crc32-ce-glue.c | 129 ++++++++++
arch/arm64/crypto/Kconfig | 6 +
arch/arm64/crypto/Makefile | 3 +
arch/arm64/crypto/crc32-ce-core.S | 246 +++++++++++++++++++
arch/arm64/crypto/crc32-ce-glue.c | 124 ++++++++++
8 files changed, 772 insertions(+)
create mode 100644 arch/arm/crypto/crc32-ce-core.S
create mode 100644 arch/arm/crypto/crc32-ce-glue.c
create mode 100644 arch/arm64/crypto/crc32-ce-core.S
create mode 100644 arch/arm64/crypto/crc32-ce-glue.c
--
2.7.4
^ permalink raw reply
* [PATCH] crypto: CTR DRBG - prevent invalid SG mappings
From: Stephan Mueller @ 2016-11-26 8:54 UTC (permalink / raw)
To: herbert; +Cc: linux-crypto
In-Reply-To: <3805767.QuT5G9UC4v@positron.chronox.de>
Hi Herbert,
as discussed in another thread, SGs must not be used with stack memory
pointers. This issue was the culprit to the error I see with the CTR DRBG. The
attached patch fixes the issue.
---8<---
When using SGs, only heap memory (memory that is valid as per
virt_addr_valid) is allowed to be referenced. The CTR DRBG used to
reference the caller-provided memory directly in an SG. In case the
caller provided stack memory pointers, the SG mapping is not considered
to be valid. In some cases, this would even cause a paging fault.
The change adds a new scratch buffer that is used in case the
caller-provided buffer is deemed not suitable for use in an SG. The
crypto operation of the CTR DRBG produces its output with that scratch
buffer.
The scratch buffer is allocated during allocation time of the CTR DRBG
as its access is protected with the DRBG mutex.
Signed-off-by: Stephan Mueller <smueller@chronox.de>
---
crypto/drbg.c | 35 +++++++++++++++++++++++++++++++----
include/crypto/drbg.h | 2 ++
2 files changed, 33 insertions(+), 4 deletions(-)
diff --git a/crypto/drbg.c b/crypto/drbg.c
index 9a95b61..cbbd19f 100644
--- a/crypto/drbg.c
+++ b/crypto/drbg.c
@@ -262,6 +262,7 @@ static int drbg_kcapi_sym_ctr(struct drbg_state *drbg,
u8 *inbuf, u32 inbuflen,
u8 *outbuf, u32 outlen);
#define DRBG_CTR_NULL_LEN 128
+#define DRBG_OUTSCRATCHLEN DRBG_CTR_NULL_LEN
/* BCC function for CTR DRBG as defined in 10.4.3 */
static int drbg_ctr_bcc(struct drbg_state *drbg,
@@ -1644,6 +1645,9 @@ static int drbg_fini_sym_kernel(struct drbg_state *drbg)
kfree(drbg->ctr_null_value_buf);
drbg->ctr_null_value = NULL;
+ kfree(drbg->outscratchpadbuf);
+ drbg->outscratchpadbuf = NULL;
+
return 0;
}
@@ -1708,6 +1712,15 @@ static int drbg_init_sym_kernel(struct drbg_state
*drbg)
drbg->ctr_null_value = (u8 *)PTR_ALIGN(drbg->ctr_null_value_buf,
alignmask + 1);
+ drbg->outscratchpadbuf = kmalloc(DRBG_OUTSCRATCHLEN + alignmask,
+ GFP_KERNEL);
+ if (!drbg->outscratchpadbuf) {
+ drbg_fini_sym_kernel(drbg);
+ return -ENOMEM;
+ }
+ drbg->outscratchpad = (u8 *)PTR_ALIGN(drbg->outscratchpadbuf,
+ alignmask + 1);
+
return alignmask;
}
@@ -1737,15 +1750,22 @@ static int drbg_kcapi_sym_ctr(struct drbg_state *drbg,
u8 *outbuf, u32 outlen)
{
struct scatterlist sg_in;
+ bool virt_addr_valid = virt_addr_valid(outbuf);
+ int ret = 0;
sg_init_one(&sg_in, inbuf, inlen);
while (outlen) {
u32 cryptlen = min_t(u32, inlen, outlen);
struct scatterlist sg_out;
- int ret;
- sg_init_one(&sg_out, outbuf, cryptlen);
+ /* If output buffer is not valid for SGL, use scratchpad */
+ if (virt_addr_valid)
+ sg_init_one(&sg_out, outbuf, cryptlen);
+ else {
+ cryptlen = min_t(u32, cryptlen, DRBG_OUTSCRATCHLEN);
+ sg_init_one(&sg_out, drbg->outscratchpad, cryptlen);
+ }
skcipher_request_set_crypt(drbg->ctr_req, &sg_in, &sg_out,
cryptlen, drbg->V);
ret = crypto_skcipher_encrypt(drbg->ctr_req);
@@ -1761,15 +1781,22 @@ static int drbg_kcapi_sym_ctr(struct drbg_state *drbg,
break;
}
default:
- return ret;
+ goto out;
}
init_completion(&drbg->ctr_completion);
+ if (!virt_addr_valid)
+ memcpy(outbuf, drbg->outscratchpad, cryptlen);
+
outlen -= cryptlen;
outbuf += cryptlen;
}
+ ret = 0;
- return 0;
+out:
+ if (!virt_addr_valid)
+ memzero_explicit(drbg->outscratchpad, DRBG_OUTSCRATCHLEN);
+ return ret;
}
#endif /* CONFIG_CRYPTO_DRBG_CTR */
diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h
index 61580b1..22f884c 100644
--- a/include/crypto/drbg.h
+++ b/include/crypto/drbg.h
@@ -124,6 +124,8 @@ struct drbg_state {
struct skcipher_request *ctr_req; /* CTR mode request handle */
__u8 *ctr_null_value_buf; /* CTR mode unaligned buffer */
__u8 *ctr_null_value; /* CTR mode aligned zero buf */
+ __u8 *outscratchpadbuf; /* CTR mode output scratchpad */
+ __u8 *outscratchpad; /* CTR mode aligned outbuf */
struct completion ctr_completion; /* CTR mode async handler */
int ctr_async_err; /* CTR mode async error */
--
2.9.3
^ permalink raw reply related
* RE: [PATCH v2 0/2] virtio-crypto: add Linux driver
From: Gonglei (Arei) @ 2016-11-26 9:38 UTC (permalink / raw)
To: Gonglei (Arei), linux-kernel@vger.kernel.org,
qemu-devel@nongnu.org, virtio-dev@lists.oasis-open.org,
virtualization@lists.linux-foundation.org,
linux-crypto@vger.kernel.org
Cc: Luonengjun, mst@redhat.com, stefanha@redhat.com, Huangweidong (C),
Wubin (H), xin.zeng@intel.com, Claudio Fontana,
herbert@gondor.apana.org.au, pasic@linux.vnet.ibm.com,
davem@davemloft.net, Zhoujian (jay, Euler), Hanweidong (Randy),
arei.gonglei@hotmail.com, cornelia.huck@de.ibm.com,
Xuquan (Quan Xu), longpeng,
"salvatore.benedetto@intel.com" <salvat
In-Reply-To: <1479802223-121104-1-git-send-email-arei.gonglei@huawei.com>
Hi,
> -----Original Message-----
> From: Gonglei (Arei)
> Sent: Tuesday, November 22, 2016 4:10 PM
> To: linux-kernel@vger.kernel.org; qemu-devel@nongnu.org;
> virtio-dev@lists.oasis-open.org; virtualization@lists.linux-foundation.org;
> linux-crypto@vger.kernel.org
> Subject: [PATCH v2 0/2] virtio-crypto: add Linux driver
>
> The virtio crypto device is a virtual cryptography device
> as well as a kind of virtual hardware accelerator for
> virtual machines. The encryption anddecryption requests
> are placed in the data queue and are ultimately handled by
> thebackend crypto accelerators. The second queue is the
> control queue used to create or destroy sessions for
> symmetric algorithms and will control some advanced features
> in the future. The virtio crypto device provides the following
> cryptoservices: CIPHER, MAC, HASH, and AEAD.
>
> For more information about virtio-crypto device, please see:
> http://qemu-project.org/Features/VirtioCrypto
>
> For better reviewing:
>
> Patch 1 introduces the little edian functions for VIRTIO_1
> devices.
>
> Patch 2 mainly includes five files:
> 1) virtio_crypto.h is the header file for virtio-crypto device,
> which is based on the virtio-crypto specification.
> 2) virtio_crypto.c is the entry of the driver module,
> which is similar with other virtio devices, such as virtio-net,
> virtio-input etc.
> 3) virtio_crypto_mgr.c is used to manage the virtio
> crypto devices in the system. We support up to 32 virtio-crypto
> devices currently. I use a global list to store the virtio crypto
> devices which refer to Intel QAT driver. Meanwhile, the file
> includs the functions of add/del/search/start/stop for virtio
> crypto devices.
> 4) virtio_crypto_common.h is a private header file for virtio
> crypto driver, includes structure definations, and function declarations.
> 5) virtio_crypto_algs.c is the realization of algs based on Linux Crypto
> Framwork,
> which can register different crypto algorithms. Currently it's only support
> AES-CBC.
> The Crypto guys can mainly focus to this file.
>
> Actually I have no idea the virtio-crypto driver should be gone in whose
> tree, Michael's or Herbert's?
>
> Would you give me a feedback? Thanks a lot!
>
Ping?
Any ideas? Thanks.
Regards,
-Gonglei
>
> v2:
> - stop doing DMA from the stack, CONFIG_VMAP_STACK=y [Salvatore]
> - convert __virtio32/64 to __le32/64 in virtio_crypto.h
> - remove VIRTIO_CRYPTO_S_STARTED based on the lastest virtio crypto spec.
> - introduces the little edian functions for VIRTIO_1 devices in patch 1.
>
> Gonglei (2):
> virtio: introduce little edian functions for virtio_cread/write#
> family
> crypto: add virtio-crypto driver
>
> MAINTAINERS | 8 +
> drivers/crypto/Kconfig | 2 +
> drivers/crypto/Makefile | 1 +
> drivers/crypto/virtio/Kconfig | 10 +
> drivers/crypto/virtio/Makefile | 5 +
> drivers/crypto/virtio/virtio_crypto.c | 444
> +++++++++++++++++++++++
> drivers/crypto/virtio/virtio_crypto_algs.c | 524
> +++++++++++++++++++++++++++
> drivers/crypto/virtio/virtio_crypto_common.h | 124 +++++++
> drivers/crypto/virtio/virtio_crypto_mgr.c | 258 +++++++++++++
> include/linux/virtio_config.h | 45 +++
> include/uapi/linux/Kbuild | 1 +
> include/uapi/linux/virtio_crypto.h | 435
> ++++++++++++++++++++++
> include/uapi/linux/virtio_ids.h | 1 +
> 13 files changed, 1858 insertions(+)
> create mode 100644 drivers/crypto/virtio/Kconfig
> create mode 100644 drivers/crypto/virtio/Makefile
> create mode 100644 drivers/crypto/virtio/virtio_crypto.c
> create mode 100644 drivers/crypto/virtio/virtio_crypto_algs.c
> create mode 100644 drivers/crypto/virtio/virtio_crypto_common.h
> create mode 100644 drivers/crypto/virtio/virtio_crypto_mgr.c
> create mode 100644 include/uapi/linux/virtio_crypto.h
>
> --
> 1.8.3.1
>
^ permalink raw reply
* [PATCH] crypto: vmx - rebuild generated asm when target changes
From: Nicholas Piggin @ 2016-11-26 4:24 UTC (permalink / raw)
To: Herbert Xu
Cc: Nicholas Piggin, Leonidas S . Barbosa, Paulo Flabiano Smorigo,
linux-crypto, Michael Ellerman, linuxppc-dev
Switching from big endian to little endian can fail to regenerate
the crypto assembly properly. Switch to using standard form of
kbuild dependency checking (i.e., use FORCE and if_changed).
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
drivers/crypto/vmx/Makefile | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/drivers/crypto/vmx/Makefile b/drivers/crypto/vmx/Makefile
index b47da00..16ab2a0 100644
--- a/drivers/crypto/vmx/Makefile
+++ b/drivers/crypto/vmx/Makefile
@@ -1,6 +1,8 @@
obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o
vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o
+targets += aesp8-ppc.S ghashp8-ppc.S
+
ifdef CONFIG_CPU_LITTLE_ENDIAN
TARGET := linux-ppc64le
else
@@ -11,13 +13,13 @@ TARGET := linux-ppc64
endif
endif
-quiet_cmd_perl = PERL $@
+quiet_cmd_perl = PERL $@
cmd_perl = $(PERL) $(<) $(TARGET) > $(@)
-$(src)/aesp8-ppc.S: $(src)/aesp8-ppc.pl
- $(call cmd,perl)
+$(src)/aesp8-ppc.S: $(src)/aesp8-ppc.pl FORCE
+ $(call if_changed,perl)
-$(src)/ghashp8-ppc.S: $(src)/ghashp8-ppc.pl
- $(call cmd,perl)
+$(src)/ghashp8-ppc.S: $(src)/ghashp8-ppc.pl FORCE
+ $(call if_changed,perl)
.PRECIOUS: $(obj)/aesp8-ppc.S $(obj)/ghashp8-ppc.S
--
2.10.2
^ permalink raw reply related
* Re: bug in blkcipher_walk code
From: Stephan Mueller @ 2016-11-25 12:43 UTC (permalink / raw)
To: herbert; +Cc: linux-crypto
In-Reply-To: <1833318.YYJALW9cvC@positron.chronox.de>
Am Freitag, 18. November 2016, 12:31:10 CET schrieb Stephan Mueller:
Hi Herbert,
> Hi Herbert,
>
> Once in a while I seem to trigger a bug in the blkcipher_walk code which I
> cannot track down. This bug happens sporadically where I assume that it has
> something to do with the memory management in the slow path of
> blkcipher_walk.
>
> I am using the CTR DRBG code that in turn uses the ctr-aes-aesni
> implementation. The bug only appears when I want to obtain a random number
> that is less than the CTR AES block size. In my particular case, I want 4
> bytes from the DRBG.
>
> The bug happens in arch/x86/crypto/aesni-intel_glue.c:ctr_crypt_final() at
> the line:
>
> memcpy(dst, keystream, nbytes);
>
> The bug looks like the following:
>
> [ 12.328676] BUG: unable to handle kernel paging request at
> ffffa17ae418b988 [ 12.328680] IP: [<ffffffff82060eea>]
> ctr_crypt+0x19a/0x1c0
> [ 12.328681] PGD 66fed067
> [ 12.328681] PUD 0
> [ 12.328681]
> [ 12.328683] Oops: 0002 [#1] SMP
> [ 12.328692] Modules linked in: bridge(+) stp llc ebtable_nat ip6table_raw
> ip6table_security ip6table_mangle iptable_raw iptable_security
> iptable_mangle ebtable_filter ebtables ip6table_filter ip6_tables
> crct10dif_pclmul crc32_pclmul ghash_clmulni_intel pcspkr i2c_piix4
> virtio_net virtio_balloon acpi_cpufreq sch_fq_codel virtio_console
> virtio_blk virtio_pci virtio_ring serio_raw crc32c_intel virtio
> [ 12.328693] CPU: 0 PID: 521 Comm: modprobe Not tainted 4.9.0-rc1+ #253
> [ 12.328694] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS
> 1.9.1-1.fc24 04/01/2014
> [ 12.328694] task: ffffa17ab8453fc0 task.stack: ffffbdafc0744000
> [ 12.328696] RIP: 0010:[<ffffffff82060eea>] [<ffffffff82060eea>]
> ctr_crypt +0x19a/0x1c0
> [ 12.328696] RSP: 0018:ffffbdafc0747a60 EFLAGS: 00010002
> [ 12.328697] RAX: 0000000032e455a6 RBX: 0000000000000004 RCX:
> 0000000000000002
> [ 12.328697] RDX: 0000000000000001 RSI: 0000000000000086 RDI:
> 0000000000000086
> [ 12.328698] RBP: ffffbdafc0747b28 R08: ffffa17abc16e900 R09:
> 0000000000000019
> [ 12.328698] R10: ffffa17a764f68b0 R11: 000000000002e918 R12:
> ffffbdafc0747b38
> [ 12.328698] R13: ffffa17a764f6840 R14: ffffa17ae418b988 R15:
> ffffbdafc0747a70
> [ 12.328699] FS: 00007f55f57a6700(0000) GS:ffffa17abfc00000(0000) knlGS:
> 0000000000000000
> [ 12.328700] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
> [ 12.328700] CR2: ffffa17ae418b988 CR3: 0000000079b26000 CR4:
> 00000000003406f0
> [ 12.328703] Stack:
> [ 12.328705] ffffa17abc16e900 ffffa17ab845fd80 2ae7e40732e455a6
> 3a224612a8f9841d
> [ 12.328706] fffffb4e81e117c0 ffffa17ab845fd80 fffffb4e829062c0
> ffffa17ae418b988
> [ 12.328707] ffffbdafc0747ba8 ffffffff00000d80 ffffffff00000004
> ffffbdafc0747bc8
> [ 12.328708] Call Trace:
> [ 12.328712] [<ffffffff823e5fd3>] __ablk_encrypt+0x43/0x50
> [ 12.328714] [<ffffffff823e6012>] ablk_encrypt+0x32/0xc0
> [ 12.328716] [<ffffffff823c4f2e>] skcipher_encrypt_ablkcipher+0x5e/0x60
> [ 12.328717] [<ffffffff823dbb80>] drbg_kcapi_sym_ctr+0xb0/0x130
> [ 12.328719] [<ffffffff823de153>] drbg_ctr_generate+0x53/0x80
>
>
> Now, the interesting part is the following: the original memory pointer that
> shall be processed by the DRBG is in my example ffffffffc018b988 -- this
> pointer is used until the DRBG invokes crypto_skcipher_encrypt. However,
> when I print out the buffer pointer that is used as dst in the memcpy of
> ctr_crypt_final, I see ffffa17ae418b988 -- i.e. the buffer that causes
> paging failure.
>
> During tracing the blkcipher_walk code I see that the slow code path is used
> when the request size is smaller than the block size. That slow code path
> allocates new memory that will be used for the dst pointer in
> ctr_crypt_final.
>
> May I ask you for checking whether the allocation and the memory pointer
> logic has an issue that would cause a paging failure?
Following up this issue, I found the location where the wrong memory pointer
is produced -- the following call tree is used:
1. set up of SGL with proper pointer
2. skcipher_encrypt_ablkcipher with SGL
3. invocation of ctr_crypt from arch/x86/crypto/aesni-intel_glue.c
4. blkcipher_walk_virt_block
5. blkcipher_walk_first
6. blkcipher_walk_next (this code does not use the code path to allocate a
page)
7. blkcipher_next_fast
walk->dst.virt.addr = walk->src.virt.addr;
-> copy src virt address into dst address pointer
Now, the diff path is used:
if (diff) {
walk->flags |= BLKCIPHER_WALK_DIFF;
blkcipher_map_dst(walk);
}
8. blkcipher_map_dst
walk->dst.virt.addr = scatterwalk_map(&walk->out);
==> this pointer is wrong
The interesting point is that step 8 gets the low and high bits right, but not
the bits in the middle:
The real data pointer for the dst buffer is ffffffffc0332988. The data pointer
used by the crypto API is ffff96a995332988 -- as often as I see the issue,
this similarity in the pointer values is always there.
Please note that the caller uses a static variable that shall be used as dst
buffer.
Thanks
Stephan
^ permalink raw reply
* 40147 linux-crypto
From: service @ 2016-11-24 19:05 UTC (permalink / raw)
To: linux-crypto
[-- Attachment #1: INFO_628278483951873_linux-crypto.zip --]
[-- Type: application/zip, Size: 4860 bytes --]
^ permalink raw reply
* Re: [PATCH 4/4] crypto: arm/crct10dif - port x86 SSE implementation to ARM
From: Ard Biesheuvel @ 2016-11-24 17:32 UTC (permalink / raw)
To: linux-crypto@vger.kernel.org, Herbert Xu,
linux-arm-kernel@lists.infradead.org, Catalin Marinas,
Will Deacon, Russell King - ARM Linux
Cc: Steve Capper, dingtinahong, yangshengkai, YueHaibing, Hanjun Guo,
Ard Biesheuvel
In-Reply-To: <1480002201-1427-5-git-send-email-ard.biesheuvel@linaro.org>
On 24 November 2016 at 15:43, Ard Biesheuvel <ard.biesheuvel@linaro.org> wrote:
> This is a straight transliteration of the Intel algorithm implemented
> using SSE and PCLMULQDQ instructions that resides under in the file
> arch/x86/crypto/crct10dif-pcl-asm_64.S.
>
> Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
> ---
> arch/arm/crypto/Kconfig | 5 +
> arch/arm/crypto/Makefile | 2 +
> arch/{arm64 => arm}/crypto/crct10dif-ce-core.S | 457 +++++++++++---------
> arch/{arm64 => arm}/crypto/crct10dif-ce-glue.c | 23 +-
> 4 files changed, 277 insertions(+), 210 deletions(-)
>
This patch needs the following hunk folded in to avoid breaking the
Thumb2 build:
"""
diff --git a/arch/arm/crypto/crct10dif-ce-core.S
b/arch/arm/crypto/crct10dif-ce-core.S
index 30168b0f8581..4fdbca94dd0c 100644
--- a/arch/arm/crypto/crct10dif-ce-core.S
+++ b/arch/arm/crypto/crct10dif-ce-core.S
@@ -152,7 +152,8 @@ CPU_LE( vrev64.8 q7, q7 )
// XOR the initial_crc value
veor.8 q0, q0, q10
- adrl ip, rk3
+ARM( adrl ip, rk3 )
+THUMB( adr ip, rk3 )
vld1.64 {q10}, [ip] // xmm10 has rk3 and rk4
// type of pmull instruction
// will determine which constant to use
"""
Updated patch(es) can be found here
https://git.kernel.org/cgit/linux/kernel/git/ardb/linux.git/log/?h=arm-crct10dif
^ permalink raw reply related
* [PATCH 3/4] crypto: arm64/crct10dif - port x86 SSE implementation to arm64
From: Ard Biesheuvel @ 2016-11-24 15:43 UTC (permalink / raw)
To: linux-crypto, herbert, linux-arm-kernel, catalin.marinas,
will.deacon, linux
Cc: steve.capper, dingtianhong, yangshengkai, yuehaibing, hanjun.guo,
Ard Biesheuvel
In-Reply-To: <1480002201-1427-1-git-send-email-ard.biesheuvel@linaro.org>
This is a straight transliteration of the Intel algorithm implemented
using SSE and PCLMULQDQ instructions that resides under in the file
arch/x86/crypto/crct10dif-pcl-asm_64.S.
Suggested-by: YueHaibing <yuehaibing@huawei.com>
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/Kconfig | 5 +
arch/arm64/crypto/Makefile | 3 +
arch/arm64/crypto/crct10dif-ce-core.S | 518 ++++++++++++++++++++
arch/arm64/crypto/crct10dif-ce-glue.c | 80 +++
4 files changed, 606 insertions(+)
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 2cf32e9887e1..1b50671ffec3 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -23,6 +23,11 @@ config CRYPTO_GHASH_ARM64_CE
depends on ARM64 && KERNEL_MODE_NEON
select CRYPTO_HASH
+config CRYPTO_CRCT10DIF_ARM64_CE
+ tristate "CRCT10DIF digest algorithm using PMULL instructions"
+ depends on ARM64 && KERNEL_MODE_NEON
+ select CRYPTO_HASH
+
config CRYPTO_AES_ARM64_CE
tristate "AES core cipher using ARMv8 Crypto Extensions"
depends on ARM64 && KERNEL_MODE_NEON
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index abb79b3cfcfe..36fd3eb4201b 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -17,6 +17,9 @@ sha2-ce-y := sha2-ce-glue.o sha2-ce-core.o
obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o
ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o
+obj-$(CONFIG_CRYPTO_CRCT10DIF_ARM64_CE) += crct10dif-ce.o
+crct10dif-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o
+
obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o
CFLAGS_aes-ce-cipher.o += -march=armv8-a+crypto
diff --git a/arch/arm64/crypto/crct10dif-ce-core.S b/arch/arm64/crypto/crct10dif-ce-core.S
new file mode 100644
index 000000000000..9148ebd3470a
--- /dev/null
+++ b/arch/arm64/crypto/crct10dif-ce-core.S
@@ -0,0 +1,518 @@
+//
+// Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
+//
+// Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License version 2 as
+// published by the Free Software Foundation.
+//
+
+//
+// Implement fast CRC-T10DIF computation with SSE and PCLMULQDQ instructions
+//
+// Copyright (c) 2013, Intel Corporation
+//
+// Authors:
+// Erdinc Ozturk <erdinc.ozturk@intel.com>
+// Vinodh Gopal <vinodh.gopal@intel.com>
+// James Guilford <james.guilford@intel.com>
+// Tim Chen <tim.c.chen@linux.intel.com>
+//
+// This software is available to you under a choice of one of two
+// licenses. You may choose to be licensed under the terms of the GNU
+// General Public License (GPL) Version 2, available from the file
+// COPYING in the main directory of this source tree, or the
+// OpenIB.org BSD license below:
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the
+// distribution.
+//
+// * Neither the name of the Intel Corporation nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+//
+// THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Function API:
+// UINT16 crc_t10dif_pcl(
+// UINT16 init_crc, //initial CRC value, 16 bits
+// const unsigned char *buf, //buffer pointer to calculate CRC on
+// UINT64 len //buffer length in bytes (64-bit data)
+// );
+//
+// Reference paper titled "Fast CRC Computation for Generic
+// Polynomials Using PCLMULQDQ Instruction"
+// URL: http://www.intel.com/content/dam/www/public/us/en/documents
+// /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
+//
+//
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+ .text
+ .cpu generic+crypto
+
+ arg1_low32 .req w0
+ arg2 .req x1
+ arg3 .req x2
+
+ vzr .req v13
+
+ENTRY(crc_t10dif_pmull)
+ stp x29, x30, [sp, #-32]!
+ mov x29, sp
+
+ movi vzr.16b, #0 // init zero register
+
+ // adjust the 16-bit initial_crc value, scale it to 32 bits
+ lsl arg1_low32, arg1_low32, #16
+
+ // check if smaller than 256
+ cmp arg3, #256
+
+ // for sizes less than 128, we can't fold 64B at a time...
+ b.lt _less_than_128
+
+ // load the initial crc value
+ // crc value does not need to be byte-reflected, but it needs
+ // to be moved to the high part of the register.
+ // because data will be byte-reflected and will align with
+ // initial crc at correct place.
+ movi v10.16b, #0
+ mov v10.s[3], arg1_low32 // initial crc
+
+ // receive the initial 64B data, xor the initial crc value
+ ld1 {v0.2d-v3.2d}, [arg2], #0x40
+ ld1 {v4.2d-v7.2d}, [arg2], #0x40
+CPU_LE( rev64 v0.16b, v0.16b )
+CPU_LE( rev64 v1.16b, v1.16b )
+CPU_LE( rev64 v2.16b, v2.16b )
+CPU_LE( rev64 v3.16b, v3.16b )
+CPU_LE( rev64 v4.16b, v4.16b )
+CPU_LE( rev64 v5.16b, v5.16b )
+CPU_LE( rev64 v6.16b, v6.16b )
+CPU_LE( rev64 v7.16b, v7.16b )
+
+ ext v0.16b, v0.16b, v0.16b, #8
+ ext v1.16b, v1.16b, v1.16b, #8
+ ext v2.16b, v2.16b, v2.16b, #8
+ ext v3.16b, v3.16b, v3.16b, #8
+ ext v4.16b, v4.16b, v4.16b, #8
+ ext v5.16b, v5.16b, v5.16b, #8
+ ext v6.16b, v6.16b, v6.16b, #8
+ ext v7.16b, v7.16b, v7.16b, #8
+
+ // XOR the initial_crc value
+ eor v0.16b, v0.16b, v10.16b
+
+ ldr q10, rk3 // xmm10 has rk3 and rk4
+ // type of pmull instruction
+ // will determine which constant to use
+
+ //
+ // we subtract 256 instead of 128 to save one instruction from the loop
+ //
+ sub arg3, arg3, #256
+
+ // at this section of the code, there is 64*x+y (0<=y<64) bytes of
+ // buffer. The _fold_64_B_loop will fold 64B at a time
+ // until we have 64+y Bytes of buffer
+
+
+ // fold 64B at a time. This section of the code folds 4 vector
+ // registers in parallel
+_fold_64_B_loop:
+
+ .macro fold64, reg1, reg2
+ ld1 {v11.2d-v12.2d}, [arg2], #0x20
+CPU_LE( rev64 v11.16b, v11.16b )
+CPU_LE( rev64 v12.16b, v12.16b )
+ ext v11.16b, v11.16b, v11.16b, #8
+ ext v12.16b, v12.16b, v12.16b, #8
+
+ pmull2 v8.1q, \reg1\().2d, v10.2d
+ pmull \reg1\().1q, \reg1\().1d, v10.1d
+ pmull2 v9.1q, \reg2\().2d, v10.2d
+ pmull \reg2\().1q, \reg2\().1d, v10.1d
+
+ eor \reg1\().16b, \reg1\().16b, v11.16b
+ eor \reg2\().16b, \reg2\().16b, v12.16b
+ eor \reg1\().16b, \reg1\().16b, v8.16b
+ eor \reg2\().16b, \reg2\().16b, v9.16b
+ .endm
+
+ fold64 v0, v1
+ fold64 v2, v3
+ fold64 v4, v5
+ fold64 v6, v7
+
+ subs arg3, arg3, #128
+
+ // check if there is another 64B in the buffer to be able to fold
+ b.ge _fold_64_B_loop
+
+ // at this point, the buffer pointer is pointing at the last y Bytes
+ // of the buffer the 64B of folded data is in 4 of the vector
+ // registers: v0, v1, v2, v3
+
+ // fold the 8 vector registers to 1 vector register with different
+ // constants
+
+ .macro fold16, rk, reg
+ ldr q10, \rk
+ pmull v8.1q, \reg\().1d, v10.1d
+ pmull2 \reg\().1q, \reg\().2d, v10.2d
+ eor v7.16b, v7.16b, v8.16b
+ eor v7.16b, v7.16b, \reg\().16b
+ .endm
+
+ fold16 rk9, v0
+ fold16 rk11, v1
+ fold16 rk13, v2
+ fold16 rk15, v3
+ fold16 rk17, v4
+ fold16 rk19, v5
+ fold16 rk1, v6
+
+ // instead of 64, we add 48 to the loop counter to save 1 instruction
+ // from the loop instead of a cmp instruction, we use the negative
+ // flag with the jl instruction
+ adds arg3, arg3, #(128-16)
+ b.lt _final_reduction_for_128
+
+ // now we have 16+y bytes left to reduce. 16 Bytes is in register v7
+ // and the rest is in memory. We can fold 16 bytes at a time if y>=16
+ // continue folding 16B at a time
+
+_16B_reduction_loop:
+ pmull v8.1q, v7.1d, v10.1d
+ pmull2 v7.1q, v7.2d, v10.2d
+ eor v7.16b, v7.16b, v8.16b
+
+ ld1 {v0.2d}, [arg2], #16
+CPU_LE( rev64 v0.16b, v0.16b )
+ ext v0.16b, v0.16b, v0.16b, #8
+ eor v7.16b, v7.16b, v0.16b
+ subs arg3, arg3, #16
+
+ // instead of a cmp instruction, we utilize the flags with the
+ // jge instruction equivalent of: cmp arg3, 16-16
+ // check if there is any more 16B in the buffer to be able to fold
+ b.ge _16B_reduction_loop
+
+ // now we have 16+z bytes left to reduce, where 0<= z < 16.
+ // first, we reduce the data in the xmm7 register
+
+_final_reduction_for_128:
+ // check if any more data to fold. If not, compute the CRC of
+ // the final 128 bits
+ adds arg3, arg3, #16
+ b.eq _128_done
+
+ // here we are getting data that is less than 16 bytes.
+ // since we know that there was data before the pointer, we can
+ // offset the input pointer before the actual point, to receive
+ // exactly 16 bytes. after that the registers need to be adjusted.
+_get_last_two_regs:
+ mov v2.16b, v7.16b
+
+ add arg2, arg2, arg3
+ sub arg2, arg2, #16
+ ld1 {v1.2d}, [arg2]
+CPU_LE( rev64 v1.16b, v1.16b )
+ ext v1.16b, v1.16b, v1.16b, #8
+
+ // get rid of the extra data that was loaded before
+ // load the shift constant
+ adr x4, tbl_shf_table + 16
+ sub x4, x4, arg3
+ ld1 {v0.16b}, [x4]
+
+ // shift v2 to the left by arg3 bytes
+ tbl v2.16b, {v2.16b}, v0.16b
+
+ // shift v7 to the right by 16-arg3 bytes
+ movi v9.16b, #0x80
+ eor v0.16b, v0.16b, v9.16b
+ tbl v7.16b, {v7.16b}, v0.16b
+
+ // blend
+ sshr v0.16b, v0.16b, #7 // convert to 8-bit mask
+ bsl v0.16b, v2.16b, v1.16b
+
+ // fold 16 Bytes
+ pmull v8.1q, v7.1d, v10.1d
+ pmull2 v7.1q, v7.2d, v10.2d
+ eor v7.16b, v7.16b, v8.16b
+ eor v7.16b, v7.16b, v0.16b
+
+_128_done:
+ // compute crc of a 128-bit value
+ ldr q10, rk5 // rk5 and rk6 in xmm10
+
+ // 64b fold
+ mov v0.16b, v7.16b
+ ext v7.16b, v7.16b, v7.16b, #8
+ pmull v7.1q, v7.1d, v10.1d
+ ext v0.16b, vzr.16b, v0.16b, #8
+ eor v7.16b, v7.16b, v0.16b
+
+ // 32b fold
+ mov v0.16b, v7.16b
+ mov v0.s[3], vzr.s[0]
+ ext v7.16b, v7.16b, vzr.16b, #12
+ ext v9.16b, v10.16b, v10.16b, #8
+ pmull v7.1q, v7.1d, v9.1d
+ eor v7.16b, v7.16b, v0.16b
+
+ // barrett reduction
+_barrett:
+ ldr q10, rk7
+ mov v0.16b, v7.16b
+ ext v7.16b, v7.16b, v7.16b, #8
+
+ pmull v7.1q, v7.1d, v10.1d
+ ext v7.16b, vzr.16b, v7.16b, #12
+ pmull2 v7.1q, v7.2d, v10.2d
+ ext v7.16b, vzr.16b, v7.16b, #12
+ eor v7.16b, v7.16b, v0.16b
+ mov w0, v7.s[1]
+
+_cleanup:
+ // scale the result back to 16 bits
+ lsr x0, x0, #16
+ ldp x29, x30, [sp], #32
+ ret
+
+ .align 4
+_less_than_128:
+
+ // check if there is enough buffer to be able to fold 16B at a time
+ cmp arg3, #32
+ b.lt _less_than_32
+
+ // now if there is, load the constants
+ ldr q10, rk1 // rk1 and rk2 in xmm10
+
+ movi v0.16b, #0
+ mov v0.s[3], arg1_low32 // get the initial crc value
+ ld1 {v7.2d}, [arg2], #0x10
+CPU_LE( rev64 v7.16b, v7.16b )
+ ext v7.16b, v7.16b, v7.16b, #8
+ eor v7.16b, v7.16b, v0.16b
+
+ // update the counter. subtract 32 instead of 16 to save one
+ // instruction from the loop
+ sub arg3, arg3, #32
+
+ b _16B_reduction_loop
+
+ .align 4
+_less_than_32:
+ cbz arg3, _cleanup
+
+ movi v0.16b, #0
+ mov v0.s[3], arg1_low32 // get the initial crc value
+
+ cmp arg3, #16
+ b.eq _exact_16_left
+ b.lt _less_than_16_left
+
+ ld1 {v7.2d}, [arg2], #0x10
+CPU_LE( rev64 v7.16b, v7.16b )
+ ext v7.16b, v7.16b, v7.16b, #8
+ eor v7.16b, v7.16b, v0.16b
+ sub arg3, arg3, #16
+ ldr q10, rk1 // rk1 and rk2 in xmm10
+ b _get_last_two_regs
+
+ .align 4
+_less_than_16_left:
+ // use stack space to load data less than 16 bytes, zero-out
+ // the 16B in memory first.
+
+ add x11, sp, #0x10
+ stp xzr, xzr, [x11]
+
+ cmp arg3, #4
+ b.lt _only_less_than_4
+
+ // backup the counter value
+ mov x9, arg3
+ tbz arg3, #3, _less_than_8_left
+
+ // load 8 Bytes
+ ldr x0, [arg2], #8
+ str x0, [x11], #8
+ sub arg3, arg3, #8
+
+_less_than_8_left:
+ tbz arg3, #2, _less_than_4_left
+
+ // load 4 Bytes
+ ldr w0, [arg2], #4
+ str w0, [x11], #4
+ sub arg3, arg3, #4
+
+_less_than_4_left:
+ tbz arg3, #1, _less_than_2_left
+
+ // load 2 Bytes
+ ldrh w0, [arg2], #2
+ strh w0, [x11], #2
+ sub arg3, arg3, #2
+
+_less_than_2_left:
+ cbz arg3, _zero_left
+
+ // load 1 Byte
+ ldrb w0, [arg2]
+ strb w0, [x11]
+
+_zero_left:
+ add x11, sp, #0x10
+ ld1 {v7.2d}, [x11]
+CPU_LE( rev64 v7.16b, v7.16b )
+ ext v7.16b, v7.16b, v7.16b, #8
+ eor v7.16b, v7.16b, v0.16b
+
+ // shl r9, 4
+ adr x0, tbl_shf_table + 16
+ sub x0, x0, x9
+ ld1 {v0.16b}, [x0]
+ movi v9.16b, #0x80
+ eor v0.16b, v0.16b, v9.16b
+ tbl v7.16b, {v7.16b}, v0.16b
+
+ b _128_done
+
+ .align 4
+_exact_16_left:
+ ld1 {v7.2d}, [arg2]
+CPU_LE( rev64 v7.16b, v7.16b )
+ ext v7.16b, v7.16b, v7.16b, #8
+ eor v7.16b, v7.16b, v0.16b // xor the initial crc value
+
+ b _128_done
+
+_only_less_than_4:
+ cmp arg3, #3
+ b.lt _only_less_than_3
+
+ // load 3 Bytes
+ ldrh w0, [arg2]
+ strh w0, [x11]
+
+ ldrb w0, [arg2, #2]
+ strb w0, [x11, #2]
+
+ ld1 {v7.2d}, [x11]
+CPU_LE( rev64 v7.16b, v7.16b )
+ ext v7.16b, v7.16b, v7.16b, #8
+ eor v7.16b, v7.16b, v0.16b
+
+ ext v7.16b, v7.16b, vzr.16b, #5
+ b _barrett
+
+_only_less_than_3:
+ cmp arg3, #2
+ b.lt _only_less_than_2
+
+ // load 2 Bytes
+ ldrh w0, [arg2]
+ strh w0, [x11]
+
+ ld1 {v7.2d}, [x11]
+CPU_LE( rev64 v7.16b, v7.16b )
+ ext v7.16b, v7.16b, v7.16b, #8
+ eor v7.16b, v7.16b, v0.16b
+
+ ext v7.16b, v7.16b, vzr.16b, #6
+ b _barrett
+
+_only_less_than_2:
+
+ // load 1 Byte
+ ldrb w0, [arg2]
+ strb w0, [x11]
+
+ ld1 {v7.2d}, [x11]
+CPU_LE( rev64 v7.16b, v7.16b )
+ ext v7.16b, v7.16b, v7.16b, #8
+ eor v7.16b, v7.16b, v0.16b
+
+ ext v7.16b, v7.16b, vzr.16b, #7
+ b _barrett
+
+ENDPROC(crc_t10dif_pmull)
+
+// precomputed constants
+// these constants are precomputed from the poly:
+// 0x8bb70000 (0x8bb7 scaled to 32 bits)
+ .align 4
+// Q = 0x18BB70000
+// rk1 = 2^(32*3) mod Q << 32
+// rk2 = 2^(32*5) mod Q << 32
+// rk3 = 2^(32*15) mod Q << 32
+// rk4 = 2^(32*17) mod Q << 32
+// rk5 = 2^(32*3) mod Q << 32
+// rk6 = 2^(32*2) mod Q << 32
+// rk7 = floor(2^64/Q)
+// rk8 = Q
+
+rk1: .octa 0x06df0000000000002d56000000000000
+rk3: .octa 0x7cf50000000000009d9d000000000000
+rk5: .octa 0x13680000000000002d56000000000000
+rk7: .octa 0x000000018bb7000000000001f65a57f8
+rk9: .octa 0xbfd6000000000000ceae000000000000
+rk11: .octa 0x713c0000000000001e16000000000000
+rk13: .octa 0x80a6000000000000f7f9000000000000
+rk15: .octa 0xe658000000000000044c000000000000
+rk17: .octa 0xa497000000000000ad18000000000000
+rk19: .octa 0xe7b50000000000006ee3000000000000
+
+tbl_shf_table:
+// use these values for shift constants for the tbl/tbx instruction
+// different alignments result in values as shown:
+// DDQ 0x008f8e8d8c8b8a898887868584838281 # shl 15 (16-1) / shr1
+// DDQ 0x01008f8e8d8c8b8a8988878685848382 # shl 14 (16-3) / shr2
+// DDQ 0x0201008f8e8d8c8b8a89888786858483 # shl 13 (16-4) / shr3
+// DDQ 0x030201008f8e8d8c8b8a898887868584 # shl 12 (16-4) / shr4
+// DDQ 0x04030201008f8e8d8c8b8a8988878685 # shl 11 (16-5) / shr5
+// DDQ 0x0504030201008f8e8d8c8b8a89888786 # shl 10 (16-6) / shr6
+// DDQ 0x060504030201008f8e8d8c8b8a898887 # shl 9 (16-7) / shr7
+// DDQ 0x07060504030201008f8e8d8c8b8a8988 # shl 8 (16-8) / shr8
+// DDQ 0x0807060504030201008f8e8d8c8b8a89 # shl 7 (16-9) / shr9
+// DDQ 0x090807060504030201008f8e8d8c8b8a # shl 6 (16-10) / shr10
+// DDQ 0x0a090807060504030201008f8e8d8c8b # shl 5 (16-11) / shr11
+// DDQ 0x0b0a090807060504030201008f8e8d8c # shl 4 (16-12) / shr12
+// DDQ 0x0c0b0a090807060504030201008f8e8d # shl 3 (16-13) / shr13
+// DDQ 0x0d0c0b0a090807060504030201008f8e # shl 2 (16-14) / shr14
+// DDQ 0x0e0d0c0b0a090807060504030201008f # shl 1 (16-15) / shr15
+
+ .byte 0x0, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87
+ .byte 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f
+ .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
+ .byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe , 0x0
diff --git a/arch/arm64/crypto/crct10dif-ce-glue.c b/arch/arm64/crypto/crct10dif-ce-glue.c
new file mode 100644
index 000000000000..d11f33dae79c
--- /dev/null
+++ b/arch/arm64/crypto/crct10dif-ce-glue.c
@@ -0,0 +1,80 @@
+/*
+ * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/cpufeature.h>
+#include <linux/crc-t10dif.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/string.h>
+
+#include <crypto/internal/hash.h>
+
+#include <asm/neon.h>
+
+asmlinkage u16 crc_t10dif_pmull(u16 init_crc, const u8 buf[], u64 len);
+
+static int crct10dif_init(struct shash_desc *desc)
+{
+ u16 *crc = shash_desc_ctx(desc);
+
+ *crc = 0;
+ return 0;
+}
+
+static int crct10dif_update(struct shash_desc *desc, const u8 *data,
+ unsigned int length)
+{
+ u16 *crc = shash_desc_ctx(desc);
+
+ kernel_neon_begin_partial(14);
+ *crc = crc_t10dif_pmull(*crc, data, length);
+ kernel_neon_end();
+
+ return 0;
+}
+
+static int crct10dif_final(struct shash_desc *desc, u8 *out)
+{
+ u16 *crc = shash_desc_ctx(desc);
+
+ *(u16 *)out = *crc;
+ return 0;
+}
+
+static struct shash_alg crc_t10dif_alg = {
+ .digestsize = CRC_T10DIF_DIGEST_SIZE,
+ .init = crct10dif_init,
+ .update = crct10dif_update,
+ .final = crct10dif_final,
+
+ .descsize = CRC_T10DIF_DIGEST_SIZE,
+ .base.cra_name = "crct10dif",
+ .base.cra_driver_name = "crct10dif-arm64-ce",
+ .base.cra_priority = 200,
+ .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
+ .base.cra_module = THIS_MODULE,
+};
+
+static int __init crc_t10dif_mod_init(void)
+{
+ return crypto_register_shash(&crc_t10dif_alg);
+}
+
+static void __exit crc_t10dif_mod_exit(void)
+{
+ crypto_unregister_shash(&crc_t10dif_alg);
+}
+
+module_cpu_feature_match(PMULL, crc_t10dif_mod_init);
+module_exit(crc_t10dif_mod_exit);
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
--
2.7.4
^ permalink raw reply related
* [PATCH 2/4] crypto: testmgr - add/enhance test cases for CRC-T10DIF
From: Ard Biesheuvel @ 2016-11-24 15:43 UTC (permalink / raw)
To: linux-crypto, herbert, linux-arm-kernel, catalin.marinas,
will.deacon, linux
Cc: steve.capper, dingtianhong, yangshengkai, yuehaibing, hanjun.guo,
Ard Biesheuvel
In-Reply-To: <1480002201-1427-1-git-send-email-ard.biesheuvel@linaro.org>
The existing test cases only exercise a small slice of the various
possible code paths through the x86 SSE/PCLMULQDQ implementation,
and the upcoming ports of it for arm64. So add one that exceeds 256
bytes in size, and convert another to a chunked test.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
crypto/testmgr.h | 70 ++++++++++++--------
1 file changed, 42 insertions(+), 28 deletions(-)
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index e64a4ef9d8ca..b7cd41b25a2a 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -1334,36 +1334,50 @@ static struct hash_testvec rmd320_tv_template[] = {
}
};
-#define CRCT10DIF_TEST_VECTORS 3
+#define CRCT10DIF_TEST_VECTORS ARRAY_SIZE(crct10dif_tv_template)
static struct hash_testvec crct10dif_tv_template[] = {
{
- .plaintext = "abc",
- .psize = 3,
-#ifdef __LITTLE_ENDIAN
- .digest = "\x3b\x44",
-#else
- .digest = "\x44\x3b",
-#endif
- }, {
- .plaintext = "1234567890123456789012345678901234567890"
- "123456789012345678901234567890123456789",
- .psize = 79,
-#ifdef __LITTLE_ENDIAN
- .digest = "\x70\x4b",
-#else
- .digest = "\x4b\x70",
-#endif
- }, {
- .plaintext =
- "abcddddddddddddddddddddddddddddddddddddddddddddddddddddd",
- .psize = 56,
-#ifdef __LITTLE_ENDIAN
- .digest = "\xe3\x9c",
-#else
- .digest = "\x9c\xe3",
-#endif
- .np = 2,
- .tap = { 28, 28 }
+ .plaintext = "abc",
+ .psize = 3,
+ .digest = (u8 *)(u16 []){ 0x443b },
+ }, {
+ .plaintext = "1234567890123456789012345678901234567890"
+ "123456789012345678901234567890123456789",
+ .psize = 79,
+ .digest = (u8 *)(u16 []){ 0x4b70 },
+ .np = 2,
+ .tap = { 63, 16 },
+ }, {
+ .plaintext = "abcdddddddddddddddddddddddddddddddddddddddd"
+ "ddddddddddddd",
+ .psize = 56,
+ .digest = (u8 *)(u16 []){ 0x9ce3 },
+ .np = 8,
+ .tap = { 1, 2, 28, 7, 6, 5, 4, 3 },
+ }, {
+ .plaintext = "1234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890"
+ "123456789012345678901234567890123456789",
+ .psize = 319,
+ .digest = (u8 *)((u16 []){ 0x44c6 }),
+ }, {
+ .plaintext = "1234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890"
+ "123456789012345678901234567890123456789",
+ .psize = 319,
+ .digest = (u8 *)((u16 []){ 0x44c6 }),
+ .np = 4,
+ .tap = { 1, 255, 57, 6 },
}
};
--
2.7.4
^ permalink raw reply related
* [PATCH 1/4] crypto: testmgr - avoid overlap in chunked tests
From: Ard Biesheuvel @ 2016-11-24 15:43 UTC (permalink / raw)
To: linux-crypto, herbert, linux-arm-kernel, catalin.marinas,
will.deacon, linux
Cc: steve.capper, dingtianhong, yangshengkai, yuehaibing, hanjun.guo,
Ard Biesheuvel
In-Reply-To: <1480002201-1427-1-git-send-email-ard.biesheuvel@linaro.org>
The IDXn offsets are chosen such that tap values (which may go up to
255) end up overlapping in the xbuf allocation. In particular, IDX1
and IDX3 are too close together, so update IDX3 to avoid this issue.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
crypto/testmgr.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 62dffa0028ac..15650597dcc9 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -62,7 +62,7 @@ int alg_test(const char *driver, const char *alg, u32 type, u32 mask)
*/
#define IDX1 32
#define IDX2 32400
-#define IDX3 1
+#define IDX3 511
#define IDX4 8193
#define IDX5 22222
#define IDX6 17101
--
2.7.4
^ permalink raw reply related
* [PATCH 0/4] crypto: CRCT10DIF support for ARM and arm64
From: Ard Biesheuvel @ 2016-11-24 15:43 UTC (permalink / raw)
To: linux-crypto, herbert, linux-arm-kernel, catalin.marinas,
will.deacon, linux
Cc: steve.capper, dingtianhong, yangshengkai, yuehaibing, hanjun.guo,
Ard Biesheuvel
First of all, apologies to Yue Haibing for stealing his thunder, to some
extent. But after reviewing (and replying to) his patch, I noticed that his
code is not original code, but simply a transliteration of the existing Intel
code that resides in arch/x86/crypto/crct10dif-pcl-asm_64.S, but with the
license and copyright statement removed.
So, if we are going to transliterate code, let's credit the original authors,
even if the resulting code does not look like the code you started out with.
Then, I noticed that we could stay *much* closer to the original, and that
there is no need for jump tables or computed gotos at all. So I got a bit
carried away, and ended up reimplementing the whole thing, for both arm and64
and ARM.
Patch #1 fixes an issue in testmgr that results in spurious false negatives
in the chunking tests if the third chunk exceeds 31 bytes.
Patch #2 expands the existing CRCT10DIF test cases, to ensure that all
code paths are actually covered.
Patch #3 is a straight transliteration of the Intel code to arm64.
Patch #4 is a straight transliteration of the Intel code to ARM. This patch
is against patch #3 (using --find-copies-harder) so that it is easy to
see how the ARM code deviates from the arm64 code.
NOTE: this code uses the 64x64->128 bit polynomial multiply instruction,
which is only available on cores that implement the v8 Crypto Extensions.
Ard Biesheuvel (4):
crypto: testmgr - avoid overlap in chunked tests
crypto: testmgr - add/enhance test cases for CRC-T10DIF
crypto: arm64/crct10dif - port x86 SSE implementation to arm64
crypto: arm/crct10dif - port x86 SSE implementation to ARM
arch/arm/crypto/Kconfig | 5 +
arch/arm/crypto/Makefile | 2 +
arch/arm/crypto/crct10dif-ce-core.S | 569 ++++++++++++++++++++
arch/arm/crypto/crct10dif-ce-glue.c | 89 +++
arch/arm64/crypto/Kconfig | 5 +
arch/arm64/crypto/Makefile | 3 +
arch/arm64/crypto/crct10dif-ce-core.S | 518 ++++++++++++++++++
arch/arm64/crypto/crct10dif-ce-glue.c | 80 +++
crypto/testmgr.c | 2 +-
crypto/testmgr.h | 70 ++-
10 files changed, 1314 insertions(+), 29 deletions(-)
create mode 100644 arch/arm/crypto/crct10dif-ce-core.S
create mode 100644 arch/arm/crypto/crct10dif-ce-glue.c
create mode 100644 arch/arm64/crypto/crct10dif-ce-core.S
create mode 100644 arch/arm64/crypto/crct10dif-ce-glue.c
--
2.7.4
^ permalink raw reply
* [PATCH 4/4] crypto: arm/crct10dif - port x86 SSE implementation to ARM
From: Ard Biesheuvel @ 2016-11-24 15:43 UTC (permalink / raw)
To: linux-crypto, herbert, linux-arm-kernel, catalin.marinas,
will.deacon, linux
Cc: steve.capper, Ard Biesheuvel, yuehaibing, hanjun.guo,
dingtianhong, yangshengkai
In-Reply-To: <1480002201-1427-1-git-send-email-ard.biesheuvel@linaro.org>
This is a straight transliteration of the Intel algorithm implemented
using SSE and PCLMULQDQ instructions that resides under in the file
arch/x86/crypto/crct10dif-pcl-asm_64.S.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm/crypto/Kconfig | 5 +
arch/arm/crypto/Makefile | 2 +
arch/{arm64 => arm}/crypto/crct10dif-ce-core.S | 457 +++++++++++---------
arch/{arm64 => arm}/crypto/crct10dif-ce-glue.c | 23 +-
4 files changed, 277 insertions(+), 210 deletions(-)
diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig
index 27ed1b1cd1d7..fce801fa52a1 100644
--- a/arch/arm/crypto/Kconfig
+++ b/arch/arm/crypto/Kconfig
@@ -120,4 +120,9 @@ config CRYPTO_GHASH_ARM_CE
that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64)
that is part of the ARMv8 Crypto Extensions
+config CRYPTO_CRCT10DIF_ARM_CE
+ tristate "CRCT10DIF digest algorithm using PMULL instructions"
+ depends on KERNEL_MODE_NEON && CRC_T10DIF
+ select CRYPTO_HASH
+
endif
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
index fc5150702b64..fc77265014b7 100644
--- a/arch/arm/crypto/Makefile
+++ b/arch/arm/crypto/Makefile
@@ -13,6 +13,7 @@ ce-obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o
ce-obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o
ce-obj-$(CONFIG_CRYPTO_SHA2_ARM_CE) += sha2-arm-ce.o
ce-obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o
+ce-obj-$(CONFIG_CRYPTO_CRCT10DIF_ARM_CE) += crct10dif-arm-ce.o
ifneq ($(ce-obj-y)$(ce-obj-m),)
ifeq ($(call as-instr,.fpu crypto-neon-fp-armv8,y,n),y)
@@ -36,6 +37,7 @@ sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o
sha2-arm-ce-y := sha2-ce-core.o sha2-ce-glue.o
aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o
ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o
+crct10dif-arm-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o
quiet_cmd_perl = PERL $@
cmd_perl = $(PERL) $(<) > $(@)
diff --git a/arch/arm64/crypto/crct10dif-ce-core.S b/arch/arm/crypto/crct10dif-ce-core.S
similarity index 60%
copy from arch/arm64/crypto/crct10dif-ce-core.S
copy to arch/arm/crypto/crct10dif-ce-core.S
index 9148ebd3470a..30168b0f8581 100644
--- a/arch/arm64/crypto/crct10dif-ce-core.S
+++ b/arch/arm/crypto/crct10dif-ce-core.S
@@ -1,5 +1,5 @@
//
-// Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
+// Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions
//
// Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
//
@@ -71,20 +71,43 @@
#include <linux/linkage.h>
#include <asm/assembler.h>
- .text
- .cpu generic+crypto
-
- arg1_low32 .req w0
- arg2 .req x1
- arg3 .req x2
+#ifdef CONFIG_CPU_ENDIAN_BE8
+#define CPU_LE(code...)
+#else
+#define CPU_LE(code...) code
+#endif
- vzr .req v13
+ .text
+ .fpu crypto-neon-fp-armv8
+
+ arg1_low32 .req r0
+ arg2 .req r1
+ arg3 .req r2
+
+ qzr .req q13
+
+ q0l .req d0
+ q0h .req d1
+ q1l .req d2
+ q1h .req d3
+ q2l .req d4
+ q2h .req d5
+ q3l .req d6
+ q3h .req d7
+ q4l .req d8
+ q4h .req d9
+ q5l .req d10
+ q5h .req d11
+ q6l .req d12
+ q6h .req d13
+ q7l .req d14
+ q7h .req d15
ENTRY(crc_t10dif_pmull)
- stp x29, x30, [sp, #-32]!
- mov x29, sp
+ push {r4, lr}
+ sub sp, sp, #0x10
- movi vzr.16b, #0 // init zero register
+ vmov.i8 qzr, #0 // init zero register
// adjust the 16-bit initial_crc value, scale it to 32 bits
lsl arg1_low32, arg1_low32, #16
@@ -93,41 +116,44 @@ ENTRY(crc_t10dif_pmull)
cmp arg3, #256
// for sizes less than 128, we can't fold 64B at a time...
- b.lt _less_than_128
+ blt _less_than_128
// load the initial crc value
// crc value does not need to be byte-reflected, but it needs
// to be moved to the high part of the register.
// because data will be byte-reflected and will align with
// initial crc at correct place.
- movi v10.16b, #0
- mov v10.s[3], arg1_low32 // initial crc
+ vmov s0, arg1_low32 // initial crc
+ vext.8 q10, qzr, q0, #4
// receive the initial 64B data, xor the initial crc value
- ld1 {v0.2d-v3.2d}, [arg2], #0x40
- ld1 {v4.2d-v7.2d}, [arg2], #0x40
-CPU_LE( rev64 v0.16b, v0.16b )
-CPU_LE( rev64 v1.16b, v1.16b )
-CPU_LE( rev64 v2.16b, v2.16b )
-CPU_LE( rev64 v3.16b, v3.16b )
-CPU_LE( rev64 v4.16b, v4.16b )
-CPU_LE( rev64 v5.16b, v5.16b )
-CPU_LE( rev64 v6.16b, v6.16b )
-CPU_LE( rev64 v7.16b, v7.16b )
-
- ext v0.16b, v0.16b, v0.16b, #8
- ext v1.16b, v1.16b, v1.16b, #8
- ext v2.16b, v2.16b, v2.16b, #8
- ext v3.16b, v3.16b, v3.16b, #8
- ext v4.16b, v4.16b, v4.16b, #8
- ext v5.16b, v5.16b, v5.16b, #8
- ext v6.16b, v6.16b, v6.16b, #8
- ext v7.16b, v7.16b, v7.16b, #8
+ vld1.64 {q0-q1}, [arg2]!
+ vld1.64 {q2-q3}, [arg2]!
+ vld1.64 {q4-q5}, [arg2]!
+ vld1.64 {q6-q7}, [arg2]!
+CPU_LE( vrev64.8 q0, q0 )
+CPU_LE( vrev64.8 q1, q1 )
+CPU_LE( vrev64.8 q2, q2 )
+CPU_LE( vrev64.8 q3, q3 )
+CPU_LE( vrev64.8 q4, q4 )
+CPU_LE( vrev64.8 q5, q5 )
+CPU_LE( vrev64.8 q6, q6 )
+CPU_LE( vrev64.8 q7, q7 )
+
+ vext.8 q0, q0, q0, #8
+ vext.8 q1, q1, q1, #8
+ vext.8 q2, q2, q2, #8
+ vext.8 q3, q3, q3, #8
+ vext.8 q4, q4, q4, #8
+ vext.8 q5, q5, q5, #8
+ vext.8 q6, q6, q6, #8
+ vext.8 q7, q7, q7, #8
// XOR the initial_crc value
- eor v0.16b, v0.16b, v10.16b
+ veor.8 q0, q0, q10
- ldr q10, rk3 // xmm10 has rk3 and rk4
+ adrl ip, rk3
+ vld1.64 {q10}, [ip] // xmm10 has rk3 and rk4
// type of pmull instruction
// will determine which constant to use
@@ -146,32 +172,32 @@ CPU_LE( rev64 v7.16b, v7.16b )
_fold_64_B_loop:
.macro fold64, reg1, reg2
- ld1 {v11.2d-v12.2d}, [arg2], #0x20
-CPU_LE( rev64 v11.16b, v11.16b )
-CPU_LE( rev64 v12.16b, v12.16b )
- ext v11.16b, v11.16b, v11.16b, #8
- ext v12.16b, v12.16b, v12.16b, #8
-
- pmull2 v8.1q, \reg1\().2d, v10.2d
- pmull \reg1\().1q, \reg1\().1d, v10.1d
- pmull2 v9.1q, \reg2\().2d, v10.2d
- pmull \reg2\().1q, \reg2\().1d, v10.1d
-
- eor \reg1\().16b, \reg1\().16b, v11.16b
- eor \reg2\().16b, \reg2\().16b, v12.16b
- eor \reg1\().16b, \reg1\().16b, v8.16b
- eor \reg2\().16b, \reg2\().16b, v9.16b
+ vld1.64 {q11-q12}, [arg2]!
+CPU_LE( vrev64.8 q11, q11 )
+CPU_LE( vrev64.8 q12, q12 )
+ vext.8 q11, q11, q11, #8
+ vext.8 q12, q12, q12, #8
+
+ vmull.p64 q8, \reg1\()h, d21
+ vmull.p64 \reg1\(), \reg1\()l, d20
+ vmull.p64 q9, \reg2\()h, d21
+ vmull.p64 \reg2\(), \reg2\()l, d20
+
+ veor.8 \reg1, \reg1, q11
+ veor.8 \reg2, \reg2, q12
+ veor.8 \reg1, \reg1, q8
+ veor.8 \reg2, \reg2, q9
.endm
- fold64 v0, v1
- fold64 v2, v3
- fold64 v4, v5
- fold64 v6, v7
+ fold64 q0, q1
+ fold64 q2, q3
+ fold64 q4, q5
+ fold64 q6, q7
subs arg3, arg3, #128
// check if there is another 64B in the buffer to be able to fold
- b.ge _fold_64_B_loop
+ bge _fold_64_B_loop
// at this point, the buffer pointer is pointing at the last y Bytes
// of the buffer the 64B of folded data is in 4 of the vector
@@ -181,46 +207,47 @@ CPU_LE( rev64 v12.16b, v12.16b )
// constants
.macro fold16, rk, reg
- ldr q10, \rk
- pmull v8.1q, \reg\().1d, v10.1d
- pmull2 \reg\().1q, \reg\().2d, v10.2d
- eor v7.16b, v7.16b, v8.16b
- eor v7.16b, v7.16b, \reg\().16b
+ vldr d20, \rk
+ vldr d21, \rk + 8
+ vmull.p64 q8, \reg\()l, d20
+ vmull.p64 \reg\(), \reg\()h, d21
+ veor.8 q7, q7, q8
+ veor.8 q7, q7, \reg
.endm
- fold16 rk9, v0
- fold16 rk11, v1
- fold16 rk13, v2
- fold16 rk15, v3
- fold16 rk17, v4
- fold16 rk19, v5
- fold16 rk1, v6
+ fold16 rk9, q0
+ fold16 rk11, q1
+ fold16 rk13, q2
+ fold16 rk15, q3
+ fold16 rk17, q4
+ fold16 rk19, q5
+ fold16 rk1, q6
// instead of 64, we add 48 to the loop counter to save 1 instruction
// from the loop instead of a cmp instruction, we use the negative
// flag with the jl instruction
adds arg3, arg3, #(128-16)
- b.lt _final_reduction_for_128
+ blt _final_reduction_for_128
// now we have 16+y bytes left to reduce. 16 Bytes is in register v7
// and the rest is in memory. We can fold 16 bytes at a time if y>=16
// continue folding 16B at a time
_16B_reduction_loop:
- pmull v8.1q, v7.1d, v10.1d
- pmull2 v7.1q, v7.2d, v10.2d
- eor v7.16b, v7.16b, v8.16b
-
- ld1 {v0.2d}, [arg2], #16
-CPU_LE( rev64 v0.16b, v0.16b )
- ext v0.16b, v0.16b, v0.16b, #8
- eor v7.16b, v7.16b, v0.16b
+ vmull.p64 q8, d14, d20
+ vmull.p64 q7, d15, d21
+ veor.8 q7, q7, q8
+
+ vld1.64 {q0}, [arg2]!
+CPU_LE( vrev64.8 q0, q0 )
+ vext.8 q0, q0, q0, #8
+ veor.8 q7, q7, q0
subs arg3, arg3, #16
// instead of a cmp instruction, we utilize the flags with the
// jge instruction equivalent of: cmp arg3, 16-16
// check if there is any more 16B in the buffer to be able to fold
- b.ge _16B_reduction_loop
+ bge _16B_reduction_loop
// now we have 16+z bytes left to reduce, where 0<= z < 16.
// first, we reduce the data in the xmm7 register
@@ -229,99 +256,104 @@ _final_reduction_for_128:
// check if any more data to fold. If not, compute the CRC of
// the final 128 bits
adds arg3, arg3, #16
- b.eq _128_done
+ beq _128_done
// here we are getting data that is less than 16 bytes.
// since we know that there was data before the pointer, we can
// offset the input pointer before the actual point, to receive
// exactly 16 bytes. after that the registers need to be adjusted.
_get_last_two_regs:
- mov v2.16b, v7.16b
+ vmov q2, q7
add arg2, arg2, arg3
sub arg2, arg2, #16
- ld1 {v1.2d}, [arg2]
-CPU_LE( rev64 v1.16b, v1.16b )
- ext v1.16b, v1.16b, v1.16b, #8
+ vld1.64 {q1}, [arg2]
+CPU_LE( vrev64.8 q1, q1 )
+ vext.8 q1, q1, q1, #8
// get rid of the extra data that was loaded before
// load the shift constant
- adr x4, tbl_shf_table + 16
- sub x4, x4, arg3
- ld1 {v0.16b}, [x4]
+ adr lr, tbl_shf_table + 16
+ sub lr, lr, arg3
+ vld1.8 {q0}, [lr]
// shift v2 to the left by arg3 bytes
- tbl v2.16b, {v2.16b}, v0.16b
+ vmov q9, q2
+ vtbl.8 d4, {d18-d19}, d0
+ vtbl.8 d5, {d18-d19}, d1
// shift v7 to the right by 16-arg3 bytes
- movi v9.16b, #0x80
- eor v0.16b, v0.16b, v9.16b
- tbl v7.16b, {v7.16b}, v0.16b
+ vmov.i8 q9, #0x80
+ veor.8 q0, q0, q9
+ vmov q9, q7
+ vtbl.8 d14, {d18-d19}, d0
+ vtbl.8 d15, {d18-d19}, d1
// blend
- sshr v0.16b, v0.16b, #7 // convert to 8-bit mask
- bsl v0.16b, v2.16b, v1.16b
+ vshr.s8 q0, q0, #7 // convert to 8-bit mask
+ vbsl.8 q0, q2, q1
// fold 16 Bytes
- pmull v8.1q, v7.1d, v10.1d
- pmull2 v7.1q, v7.2d, v10.2d
- eor v7.16b, v7.16b, v8.16b
- eor v7.16b, v7.16b, v0.16b
+ vmull.p64 q8, d14, d20
+ vmull.p64 q7, d15, d21
+ veor.8 q7, q7, q8
+ veor.8 q7, q7, q0
_128_done:
// compute crc of a 128-bit value
- ldr q10, rk5 // rk5 and rk6 in xmm10
+ vldr d20, rk5
+ vldr d21, rk6 // rk5 and rk6 in xmm10
// 64b fold
- mov v0.16b, v7.16b
- ext v7.16b, v7.16b, v7.16b, #8
- pmull v7.1q, v7.1d, v10.1d
- ext v0.16b, vzr.16b, v0.16b, #8
- eor v7.16b, v7.16b, v0.16b
+ vmov q0, q7
+ vmull.p64 q7, d15, d20
+ vext.8 q0, qzr, q0, #8
+ veor.8 q7, q7, q0
// 32b fold
- mov v0.16b, v7.16b
- mov v0.s[3], vzr.s[0]
- ext v7.16b, v7.16b, vzr.16b, #12
- ext v9.16b, v10.16b, v10.16b, #8
- pmull v7.1q, v7.1d, v9.1d
- eor v7.16b, v7.16b, v0.16b
+ veor.8 d1, d1, d1
+ vmov d0, d14
+ vmov s2, s30
+ vext.8 q7, q7, qzr, #12
+ vmull.p64 q7, d14, d21
+ veor.8 q7, q7, q0
// barrett reduction
_barrett:
- ldr q10, rk7
- mov v0.16b, v7.16b
- ext v7.16b, v7.16b, v7.16b, #8
+ vldr d20, rk7
+ vldr d21, rk8
+ vmov.8 q0, q7
- pmull v7.1q, v7.1d, v10.1d
- ext v7.16b, vzr.16b, v7.16b, #12
- pmull2 v7.1q, v7.2d, v10.2d
- ext v7.16b, vzr.16b, v7.16b, #12
- eor v7.16b, v7.16b, v0.16b
- mov w0, v7.s[1]
+ vmull.p64 q7, d15, d20
+ vext.8 q7, qzr, q7, #12
+ vmull.p64 q7, d15, d21
+ vext.8 q7, qzr, q7, #12
+ veor.8 q7, q7, q0
+ vmov r0, s29
_cleanup:
// scale the result back to 16 bits
- lsr x0, x0, #16
- ldp x29, x30, [sp], #32
- ret
+ lsr r0, r0, #16
+ add sp, sp, #0x10
+ pop {r4, pc}
.align 4
_less_than_128:
// check if there is enough buffer to be able to fold 16B at a time
cmp arg3, #32
- b.lt _less_than_32
+ blt _less_than_32
// now if there is, load the constants
- ldr q10, rk1 // rk1 and rk2 in xmm10
+ vldr d20, rk1
+ vldr d21, rk2 // rk1 and rk2 in xmm10
- movi v0.16b, #0
- mov v0.s[3], arg1_low32 // get the initial crc value
- ld1 {v7.2d}, [arg2], #0x10
-CPU_LE( rev64 v7.16b, v7.16b )
- ext v7.16b, v7.16b, v7.16b, #8
- eor v7.16b, v7.16b, v0.16b
+ vmov.i8 q0, #0
+ vmov s3, arg1_low32 // get the initial crc value
+ vld1.64 {q7}, [arg2]!
+CPU_LE( vrev64.8 q7, q7 )
+ vext.8 q7, q7, q7, #8
+ veor.8 q7, q7, q0
// update the counter. subtract 32 instead of 16 to save one
// instruction from the loop
@@ -331,21 +363,23 @@ CPU_LE( rev64 v7.16b, v7.16b )
.align 4
_less_than_32:
- cbz arg3, _cleanup
+ teq arg3, #0
+ beq _cleanup
- movi v0.16b, #0
- mov v0.s[3], arg1_low32 // get the initial crc value
+ vmov.i8 q0, #0
+ vmov s3, arg1_low32 // get the initial crc value
cmp arg3, #16
- b.eq _exact_16_left
- b.lt _less_than_16_left
+ beq _exact_16_left
+ blt _less_than_16_left
- ld1 {v7.2d}, [arg2], #0x10
-CPU_LE( rev64 v7.16b, v7.16b )
- ext v7.16b, v7.16b, v7.16b, #8
- eor v7.16b, v7.16b, v0.16b
+ vld1.64 {q7}, [arg2]!
+CPU_LE( vrev64.8 q7, q7 )
+ vext.8 q7, q7, q7, #8
+ veor.8 q7, q7, q0
sub arg3, arg3, #16
- ldr q10, rk1 // rk1 and rk2 in xmm10
+ vldr d20, rk1
+ vldr d21, rk2 // rk1 and rk2 in xmm10
b _get_last_two_regs
.align 4
@@ -353,117 +387,124 @@ _less_than_16_left:
// use stack space to load data less than 16 bytes, zero-out
// the 16B in memory first.
- add x11, sp, #0x10
- stp xzr, xzr, [x11]
+ vst1.8 {qzr}, [sp]
+ mov ip, sp
cmp arg3, #4
- b.lt _only_less_than_4
+ blt _only_less_than_4
// backup the counter value
- mov x9, arg3
- tbz arg3, #3, _less_than_8_left
+ mov lr, arg3
+ cmp arg3, #8
+ blt _less_than_8_left
// load 8 Bytes
- ldr x0, [arg2], #8
- str x0, [x11], #8
+ ldr r0, [arg2], #4
+ ldr r3, [arg2], #4
+ str r0, [ip], #4
+ str r3, [ip], #4
sub arg3, arg3, #8
_less_than_8_left:
- tbz arg3, #2, _less_than_4_left
+ cmp arg3, #4
+ blt _less_than_4_left
// load 4 Bytes
- ldr w0, [arg2], #4
- str w0, [x11], #4
+ ldr r0, [arg2], #4
+ str r0, [ip], #4
sub arg3, arg3, #4
_less_than_4_left:
- tbz arg3, #1, _less_than_2_left
+ cmp arg3, #2
+ blt _less_than_2_left
// load 2 Bytes
- ldrh w0, [arg2], #2
- strh w0, [x11], #2
+ ldrh r0, [arg2], #2
+ strh r0, [ip], #2
sub arg3, arg3, #2
_less_than_2_left:
- cbz arg3, _zero_left
+ cmp arg3, #1
+ blt _zero_left
// load 1 Byte
- ldrb w0, [arg2]
- strb w0, [x11]
+ ldrb r0, [arg2]
+ strb r0, [ip]
_zero_left:
- add x11, sp, #0x10
- ld1 {v7.2d}, [x11]
-CPU_LE( rev64 v7.16b, v7.16b )
- ext v7.16b, v7.16b, v7.16b, #8
- eor v7.16b, v7.16b, v0.16b
+ vld1.64 {q7}, [sp]
+CPU_LE( vrev64.8 q7, q7 )
+ vext.8 q7, q7, q7, #8
+ veor.8 q7, q7, q0
// shl r9, 4
- adr x0, tbl_shf_table + 16
- sub x0, x0, x9
- ld1 {v0.16b}, [x0]
- movi v9.16b, #0x80
- eor v0.16b, v0.16b, v9.16b
- tbl v7.16b, {v7.16b}, v0.16b
+ adr ip, tbl_shf_table + 16
+ sub ip, ip, lr
+ vld1.8 {q0}, [ip]
+ vmov.i8 q9, #0x80
+ veor.8 q0, q0, q9
+ vmov q9, q7
+ vtbl.8 d14, {d18-d19}, d0
+ vtbl.8 d15, {d18-d19}, d1
b _128_done
.align 4
_exact_16_left:
- ld1 {v7.2d}, [arg2]
-CPU_LE( rev64 v7.16b, v7.16b )
- ext v7.16b, v7.16b, v7.16b, #8
- eor v7.16b, v7.16b, v0.16b // xor the initial crc value
+ vld1.64 {q7}, [arg2]
+CPU_LE( vrev64.8 q7, q7 )
+ vext.8 q7, q7, q7, #8
+ veor.8 q7, q7, q0 // xor the initial crc value
b _128_done
_only_less_than_4:
cmp arg3, #3
- b.lt _only_less_than_3
+ blt _only_less_than_3
// load 3 Bytes
- ldrh w0, [arg2]
- strh w0, [x11]
+ ldrh r0, [arg2]
+ strh r0, [ip]
- ldrb w0, [arg2, #2]
- strb w0, [x11, #2]
+ ldrb r0, [arg2, #2]
+ strb r0, [ip, #2]
- ld1 {v7.2d}, [x11]
-CPU_LE( rev64 v7.16b, v7.16b )
- ext v7.16b, v7.16b, v7.16b, #8
- eor v7.16b, v7.16b, v0.16b
+ vld1.64 {q7}, [ip]
+CPU_LE( vrev64.8 q7, q7 )
+ vext.8 q7, q7, q7, #8
+ veor.8 q7, q7, q0
- ext v7.16b, v7.16b, vzr.16b, #5
+ vext.8 q7, q7, qzr, #5
b _barrett
_only_less_than_3:
cmp arg3, #2
- b.lt _only_less_than_2
+ blt _only_less_than_2
// load 2 Bytes
- ldrh w0, [arg2]
- strh w0, [x11]
+ ldrh r0, [arg2]
+ strh r0, [ip]
- ld1 {v7.2d}, [x11]
-CPU_LE( rev64 v7.16b, v7.16b )
- ext v7.16b, v7.16b, v7.16b, #8
- eor v7.16b, v7.16b, v0.16b
+ vld1.64 {q7}, [ip]
+CPU_LE( vrev64.8 q7, q7 )
+ vext.8 q7, q7, q7, #8
+ veor.8 q7, q7, q0
- ext v7.16b, v7.16b, vzr.16b, #6
+ vext.8 q7, q7, qzr, #6
b _barrett
_only_less_than_2:
// load 1 Byte
- ldrb w0, [arg2]
- strb w0, [x11]
+ ldrb r0, [arg2]
+ strb r0, [ip]
- ld1 {v7.2d}, [x11]
-CPU_LE( rev64 v7.16b, v7.16b )
- ext v7.16b, v7.16b, v7.16b, #8
- eor v7.16b, v7.16b, v0.16b
+ vld1.64 {q7}, [ip]
+CPU_LE( vrev64.8 q7, q7 )
+ vext.8 q7, q7, q7, #8
+ veor.8 q7, q7, q0
- ext v7.16b, v7.16b, vzr.16b, #7
+ vext.8 q7, q7, qzr, #7
b _barrett
ENDPROC(crc_t10dif_pmull)
@@ -482,16 +523,26 @@ ENDPROC(crc_t10dif_pmull)
// rk7 = floor(2^64/Q)
// rk8 = Q
-rk1: .octa 0x06df0000000000002d56000000000000
-rk3: .octa 0x7cf50000000000009d9d000000000000
-rk5: .octa 0x13680000000000002d56000000000000
-rk7: .octa 0x000000018bb7000000000001f65a57f8
-rk9: .octa 0xbfd6000000000000ceae000000000000
-rk11: .octa 0x713c0000000000001e16000000000000
-rk13: .octa 0x80a6000000000000f7f9000000000000
-rk15: .octa 0xe658000000000000044c000000000000
-rk17: .octa 0xa497000000000000ad18000000000000
-rk19: .octa 0xe7b50000000000006ee3000000000000
+rk1: .quad 0x2d56000000000000
+rk2: .quad 0x06df000000000000
+rk3: .quad 0x9d9d000000000000
+rk4: .quad 0x7cf5000000000000
+rk5: .quad 0x2d56000000000000
+rk6: .quad 0x1368000000000000
+rk7: .quad 0x00000001f65a57f8
+rk8: .quad 0x000000018bb70000
+rk9: .quad 0xceae000000000000
+rk10: .quad 0xbfd6000000000000
+rk11: .quad 0x1e16000000000000
+rk12: .quad 0x713c000000000000
+rk13: .quad 0xf7f9000000000000
+rk14: .quad 0x80a6000000000000
+rk15: .quad 0x044c000000000000
+rk16: .quad 0xe658000000000000
+rk17: .quad 0xad18000000000000
+rk18: .quad 0xa497000000000000
+rk19: .quad 0x6ee3000000000000
+rk20: .quad 0xe7b5000000000000
tbl_shf_table:
// use these values for shift constants for the tbl/tbx instruction
diff --git a/arch/arm64/crypto/crct10dif-ce-glue.c b/arch/arm/crypto/crct10dif-ce-glue.c
similarity index 76%
copy from arch/arm64/crypto/crct10dif-ce-glue.c
copy to arch/arm/crypto/crct10dif-ce-glue.c
index d11f33dae79c..e717538d902c 100644
--- a/arch/arm64/crypto/crct10dif-ce-glue.c
+++ b/arch/arm/crypto/crct10dif-ce-glue.c
@@ -1,5 +1,5 @@
/*
- * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
+ * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions
*
* Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
*
@@ -8,7 +8,6 @@
* published by the Free Software Foundation.
*/
-#include <linux/cpufeature.h>
#include <linux/crc-t10dif.h>
#include <linux/init.h>
#include <linux/kernel.h>
@@ -18,6 +17,7 @@
#include <crypto/internal/hash.h>
#include <asm/neon.h>
+#include <asm/simd.h>
asmlinkage u16 crc_t10dif_pmull(u16 init_crc, const u8 buf[], u64 len);
@@ -34,9 +34,13 @@ static int crct10dif_update(struct shash_desc *desc, const u8 *data,
{
u16 *crc = shash_desc_ctx(desc);
- kernel_neon_begin_partial(14);
- *crc = crc_t10dif_pmull(*crc, data, length);
- kernel_neon_end();
+ if (may_use_simd()) {
+ kernel_neon_begin();
+ *crc = crc_t10dif_pmull(*crc, data, length);
+ kernel_neon_end();
+ } else {
+ *crc = crc_t10dif_generic(*crc, data, length);
+ }
return 0;
}
@@ -57,7 +61,7 @@ static struct shash_alg crc_t10dif_alg = {
.descsize = CRC_T10DIF_DIGEST_SIZE,
.base.cra_name = "crct10dif",
- .base.cra_driver_name = "crct10dif-arm64-ce",
+ .base.cra_driver_name = "crct10dif-arm-ce",
.base.cra_priority = 200,
.base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
.base.cra_module = THIS_MODULE,
@@ -65,6 +69,9 @@ static struct shash_alg crc_t10dif_alg = {
static int __init crc_t10dif_mod_init(void)
{
+ if (!(elf_hwcap2 & HWCAP2_PMULL))
+ return -ENODEV;
+
return crypto_register_shash(&crc_t10dif_alg);
}
@@ -73,8 +80,10 @@ static void __exit crc_t10dif_mod_exit(void)
crypto_unregister_shash(&crc_t10dif_alg);
}
-module_cpu_feature_match(PMULL, crc_t10dif_mod_init);
+module_init(crc_t10dif_mod_init);
module_exit(crc_t10dif_mod_exit);
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("crct10dif");
+MODULE_ALIAS_CRYPTO("crct10dif-arm-ce");
--
2.7.4
^ permalink raw reply related
* [PATCH 2/2] mpi: Fix NULL ptr dereference in mpi_powm() [ver #3]
From: David Howells @ 2016-11-24 13:23 UTC (permalink / raw)
To: jmorris
Cc: Dmitry Kasatkin, linux-kernel, stable, dhowells,
linux-security-module, keyrings, linux-crypto, linux-ima-devel,
Andrey Ryabinin
In-Reply-To: <147999377574.9697.16315343355948647181.stgit@warthog.procyon.org.uk>
From: Andrey Ryabinin <aryabinin@virtuozzo.com>
This fixes CVE-2016-8650.
If mpi_powm() is given a zero exponent, it wants to immediately return
either 1 or 0, depending on the modulus. However, if the result was
initalised with zero limb space, no limbs space is allocated and a
NULL-pointer exception ensues.
Fix this by allocating a minimal amount of limb space for the result when
the 0-exponent case when the result is 1 and not touching the limb space
when the result is 0.
This affects the use of RSA keys and X.509 certificates that carry them.
BUG: unable to handle kernel NULL pointer dereference at (null)
IP: [<ffffffff8138ce5d>] mpi_powm+0x32/0x7e6
PGD 0
Oops: 0002 [#1] SMP
Modules linked in:
CPU: 3 PID: 3014 Comm: keyctl Not tainted 4.9.0-rc6-fscache+ #278
Hardware name: ASUS All Series/H97-PLUS, BIOS 2306 10/09/2014
task: ffff8804011944c0 task.stack: ffff880401294000
RIP: 0010:[<ffffffff8138ce5d>] [<ffffffff8138ce5d>] mpi_powm+0x32/0x7e6
RSP: 0018:ffff880401297ad8 EFLAGS: 00010212
RAX: 0000000000000000 RBX: ffff88040868bec0 RCX: ffff88040868bba0
RDX: ffff88040868b260 RSI: ffff88040868bec0 RDI: ffff88040868bee0
RBP: ffff880401297ba8 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000047 R11: ffffffff8183b210 R12: 0000000000000000
R13: ffff8804087c7600 R14: 000000000000001f R15: ffff880401297c50
FS: 00007f7a7918c700(0000) GS:ffff88041fb80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000000000 CR3: 0000000401250000 CR4: 00000000001406e0
Stack:
ffff88040868bec0 0000000000000020 ffff880401297b00 ffffffff81376cd4
0000000000000100 ffff880401297b10 ffffffff81376d12 ffff880401297b30
ffffffff81376f37 0000000000000100 0000000000000000 ffff880401297ba8
Call Trace:
[<ffffffff81376cd4>] ? __sg_page_iter_next+0x43/0x66
[<ffffffff81376d12>] ? sg_miter_get_next_page+0x1b/0x5d
[<ffffffff81376f37>] ? sg_miter_next+0x17/0xbd
[<ffffffff8138ba3a>] ? mpi_read_raw_from_sgl+0xf2/0x146
[<ffffffff8132a95c>] rsa_verify+0x9d/0xee
[<ffffffff8132acca>] ? pkcs1pad_sg_set_buf+0x2e/0xbb
[<ffffffff8132af40>] pkcs1pad_verify+0xc0/0xe1
[<ffffffff8133cb5e>] public_key_verify_signature+0x1b0/0x228
[<ffffffff8133d974>] x509_check_for_self_signed+0xa1/0xc4
[<ffffffff8133cdde>] x509_cert_parse+0x167/0x1a1
[<ffffffff8133d609>] x509_key_preparse+0x21/0x1a1
[<ffffffff8133c3d7>] asymmetric_key_preparse+0x34/0x61
[<ffffffff812fc9f3>] key_create_or_update+0x145/0x399
[<ffffffff812fe227>] SyS_add_key+0x154/0x19e
[<ffffffff81001c2b>] do_syscall_64+0x80/0x191
[<ffffffff816825e4>] entry_SYSCALL64_slow_path+0x25/0x25
Code: 56 41 55 41 54 53 48 81 ec a8 00 00 00 44 8b 71 04 8b 42 04 4c 8b 67 18 45 85 f6 89 45 80 0f 84 b4 06 00 00 85 c0 75 2f 41 ff ce <49> c7 04 24 01 00 00 00 b0 01 75 0b 48 8b 41 18 48 83 38 01 0f
RIP [<ffffffff8138ce5d>] mpi_powm+0x32/0x7e6
RSP <ffff880401297ad8>
CR2: 0000000000000000
---[ end trace d82015255d4a5d8d ]---
Basically, this is a backport of a libgcrypt patch:
http://git.gnupg.org/cgi-bin/gitweb.cgi?p=libgcrypt.git;a=patch;h=6e1adb05d290aeeb1c230c763970695f4a538526
Fixes: cdec9cb5167a ("crypto: GnuPG based MPI lib - source files (part 1)")
Signed-off-by: Andrey Ryabinin <aryabinin@virtuozzo.com>
Signed-off-by: David Howells <dhowells@redhat.com>
cc: Dmitry Kasatkin <dmitry.kasatkin@gmail.com>
cc: linux-ima-devel@lists.sourceforge.net
cc: stable@vger.kernel.org
---
lib/mpi/mpi-pow.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lib/mpi/mpi-pow.c b/lib/mpi/mpi-pow.c
index 5464c8744ea9..e24388a863a7 100644
--- a/lib/mpi/mpi-pow.c
+++ b/lib/mpi/mpi-pow.c
@@ -64,8 +64,13 @@ int mpi_powm(MPI res, MPI base, MPI exp, MPI mod)
if (!esize) {
/* Exponent is zero, result is 1 mod MOD, i.e., 1 or 0
* depending on if MOD equals 1. */
- rp[0] = 1;
res->nlimbs = (msize == 1 && mod->d[0] == 1) ? 0 : 1;
+ if (res->nlimbs) {
+ if (mpi_resize(res, 1) < 0)
+ goto enomem;
+ rp = res->d;
+ rp[0] = 1;
+ }
res->sign = 0;
goto leave;
}
^ permalink raw reply related
* [PATCH 1/2] X.509: Fix double free in x509_cert_parse() [ver #3]
From: David Howells @ 2016-11-24 13:23 UTC (permalink / raw)
To: jmorris
Cc: linux-kernel, stable, dhowells, linux-security-module, keyrings,
linux-crypto, Andrey Ryabinin
In-Reply-To: <147999377574.9697.16315343355948647181.stgit@warthog.procyon.org.uk>
From: Andrey Ryabinin <aryabinin@virtuozzo.com>
We shouldn't free cert->pub->key in x509_cert_parse() because
x509_free_certificate() also does this:
BUG: Double free or freeing an invalid pointer
...
Call Trace:
[<ffffffff81896c20>] dump_stack+0x63/0x83
[<ffffffff81356571>] kasan_object_err+0x21/0x70
[<ffffffff81356ed9>] kasan_report_double_free+0x49/0x60
[<ffffffff813561ad>] kasan_slab_free+0x9d/0xc0
[<ffffffff81350b7a>] kfree+0x8a/0x1a0
[<ffffffff81844fbf>] public_key_free+0x1f/0x30
[<ffffffff818455d4>] x509_free_certificate+0x24/0x90
[<ffffffff818460bc>] x509_cert_parse+0x2bc/0x300
[<ffffffff81846cae>] x509_key_preparse+0x3e/0x330
[<ffffffff818444cf>] asymmetric_key_preparse+0x6f/0x100
[<ffffffff8178bec0>] key_create_or_update+0x260/0x5f0
[<ffffffff8178e6d9>] SyS_add_key+0x199/0x2a0
[<ffffffff821d823b>] entry_SYSCALL_64_fastpath+0x1e/0xad
Object at ffff880110bd1900, in cache kmalloc-512 size: 512
....
Freed:
PID = 2579
[<ffffffff8104283b>] save_stack_trace+0x1b/0x20
[<ffffffff813558f6>] save_stack+0x46/0xd0
[<ffffffff81356183>] kasan_slab_free+0x73/0xc0
[<ffffffff81350b7a>] kfree+0x8a/0x1a0
[<ffffffff818460a3>] x509_cert_parse+0x2a3/0x300
[<ffffffff81846cae>] x509_key_preparse+0x3e/0x330
[<ffffffff818444cf>] asymmetric_key_preparse+0x6f/0x100
[<ffffffff8178bec0>] key_create_or_update+0x260/0x5f0
[<ffffffff8178e6d9>] SyS_add_key+0x199/0x2a0
[<ffffffff821d823b>] entry_SYSCALL_64_fastpath+0x1e/0xad
Fixes: db6c43bd2132 ("crypto: KEYS: convert public key and digsig asym to the akcipher api")
Signed-off-by: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: David Howells <dhowells@redhat.com>
---
crypto/asymmetric_keys/x509_cert_parser.c | 1 -
1 file changed, 1 deletion(-)
diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c
index 865f46ea724f..c80765b211cf 100644
--- a/crypto/asymmetric_keys/x509_cert_parser.c
+++ b/crypto/asymmetric_keys/x509_cert_parser.c
@@ -133,7 +133,6 @@ struct x509_certificate *x509_cert_parse(const void *data, size_t datalen)
return cert;
error_decode:
- kfree(cert->pub->key);
kfree(ctx);
error_no_ctx:
x509_free_certificate(cert);
^ permalink raw reply related
* [PATCH 0/2] KEYS: Fixes [ver #3]
From: David Howells @ 2016-11-24 13:22 UTC (permalink / raw)
To: jmorris
Cc: dhowells, linux-security-module, keyrings, linux-kernel,
linux-crypto
Hi James,
Can you pull these patches please and pass them on to Linus? They include
the following:
(1) Fix mpi_powm()'s handling of a number with a zero exponent [CVE-2016-8650].
(2) Fix double free in X.509 error handling.
Ver #3:
- Integrate my and Andrey's patches for mpi_powm() and use mpi_resize()
instead of RESIZE_IF_NEEDED() - the latter adds a duplicate check into
the execution path of a trivial case we don't normally expect to be
taken.
Ver #2:
- Use RESIZE_IF_NEEDED() to conditionally resize the result rather than
manually doing this.
The patches can be found here also:
http://git.kernel.org/cgit/linux/kernel/git/dhowells/linux-fs.git/log/?h=keys-fixes
Tagged thusly:
git://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git
keys-fixes-20161124-3
David
---
Andrey Ryabinin (2):
X.509: Fix double free in x509_cert_parse()
mpi: Fix NULL ptr dereference in mpi_powm()
crypto/asymmetric_keys/x509_cert_parser.c | 1 -
lib/mpi/mpi-pow.c | 7 ++++++-
2 files changed, 6 insertions(+), 2 deletions(-)
^ permalink raw reply
* 53486 linux-crypto
From: e.camilla.johansson @ 2016-11-24 6:32 UTC (permalink / raw)
To: linux-crypto
[-- Attachment #1: INFO_93352944885680_linux-crypto.zip --]
[-- Type: application/zip, Size: 2549 bytes --]
^ permalink raw reply
* [PATCH] crypto: acomp - don't use stack buffer in test_acomp()
From: Eric Biggers @ 2016-11-23 18:24 UTC (permalink / raw)
To: linux-crypto
Cc: Giovanni Cabiddu, Herbert Xu, David S. Miller, Andy Lutomirski,
Eric Biggers
With virtually-mapped stacks (CONFIG_VMAP_STACK=y), using the
scatterlist crypto API with stack buffers is not allowed, and with
appropriate debugging options will cause the
'BUG_ON(!virt_addr_valid(buf));' in sg_set_buf() to be triggered.
Use a heap buffer instead.
Fixes: d7db7a882deb ("crypto: acomp - update testmgr with support for acomp")
Signed-off-by: Eric Biggers <ebiggers@google.com>
---
crypto/testmgr.c | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index ded50b6..aca1b7b 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -1448,17 +1448,21 @@ static int test_acomp(struct crypto_acomp *tfm, struct comp_testvec *ctemplate,
{
const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm));
unsigned int i;
- char output[COMP_BUF_SIZE];
+ char *output;
int ret;
struct scatterlist src, dst;
struct acomp_req *req;
struct tcrypt_result result;
+ output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
+ if (!output)
+ return -ENOMEM;
+
for (i = 0; i < ctcount; i++) {
unsigned int dlen = COMP_BUF_SIZE;
int ilen = ctemplate[i].inlen;
- memset(output, 0, sizeof(output));
+ memset(output, 0, dlen);
init_completion(&result.completion);
sg_init_one(&src, ctemplate[i].input, ilen);
sg_init_one(&dst, output, dlen);
@@ -1507,7 +1511,7 @@ static int test_acomp(struct crypto_acomp *tfm, struct comp_testvec *ctemplate,
unsigned int dlen = COMP_BUF_SIZE;
int ilen = dtemplate[i].inlen;
- memset(output, 0, sizeof(output));
+ memset(output, 0, dlen);
init_completion(&result.completion);
sg_init_one(&src, dtemplate[i].input, ilen);
sg_init_one(&dst, output, dlen);
@@ -1555,6 +1559,7 @@ static int test_acomp(struct crypto_acomp *tfm, struct comp_testvec *ctemplate,
ret = 0;
out:
+ kfree(output);
return ret;
}
--
2.8.0.rc3.226.g39d4020
^ permalink raw reply related
* [PATCH] mpi: Fix NULL ptr dereference in mpi_powm()
From: Andrey Ryabinin @ 2016-11-23 16:28 UTC (permalink / raw)
To: Herbert Xu
Cc: David S . Miller, Nicolai Stange, linux-crypto, linux-kernel,
Andrey Ryabinin, stable
Parsing certain certificates (see [1]) triggers NULL-ptr
dereference in mpi_powm():
BUG: unable to handle kernel NULL pointer dereference at (null)
IP: [<ffffffff818eb118>] mpi_powm+0xf8/0x10b0
...
Call Trace:
[<ffffffff817e06a6>] _rsa_dec.isra.2+0x66/0x80
[<ffffffff817e07c3>] rsa_verify+0x103/0x1c0
[<ffffffff817e1683>] pkcs1pad_verify+0x1c3/0x220
[<ffffffff8184549a>] public_key_verify_signature+0x3fa/0x4d0
[<ffffffff818473c7>] x509_check_for_self_signed+0x167/0x1e0
[<ffffffff8184607e>] x509_cert_parse+0x27e/0x300
[<ffffffff81846cae>] x509_key_preparse+0x3e/0x330
[<ffffffff818444cf>] asymmetric_key_preparse+0x6f/0x100
[<ffffffff8178bec0>] key_create_or_update+0x260/0x5f0
[<ffffffff8178e6d9>] SyS_add_key+0x199/0x2a0
[<ffffffff821d823b>] entry_SYSCALL_64_fastpath+0x1e/0xad
This happens because mpi_alloc(0) doesn't allocate the limb space.
Fix this by allocating the result if needed.
Basically, this is a backport of libgcrypt patch [2].
[1] http://seclists.org/fulldisclosure/2016/Nov/76
[2] http://git.gnupg.org/cgi-bin/gitweb.cgi?p=libgcrypt.git;a=patch;h=6e1adb05d290aeeb1c230c763970695f4a538526
Fixes: cdec9cb5167a ("crypto: GnuPG based MPI lib - source files (part 1)")
Signed-off-by: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: <stable@vger.kernel.org>
---
lib/mpi/mpi-pow.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lib/mpi/mpi-pow.c b/lib/mpi/mpi-pow.c
index 5464c87..9e28d12 100644
--- a/lib/mpi/mpi-pow.c
+++ b/lib/mpi/mpi-pow.c
@@ -64,8 +64,13 @@ int mpi_powm(MPI res, MPI base, MPI exp, MPI mod)
if (!esize) {
/* Exponent is zero, result is 1 mod MOD, i.e., 1 or 0
* depending on if MOD equals 1. */
- rp[0] = 1;
res->nlimbs = (msize == 1 && mod->d[0] == 1) ? 0 : 1;
+ if (res->nlimbs) {
+ if (RESIZE_IF_NEEDED(res, 1) < 0)
+ goto enomem;
+ rp = res->d;
+ rp[0] = 1;
+ }
res->sign = 0;
goto leave;
}
--
2.7.3
^ permalink raw reply related
* [PATCH] X.509: Fix double free in x509_cert_parse()
From: Andrey Ryabinin @ 2016-11-23 16:16 UTC (permalink / raw)
To: David Howells
Cc: Tadeusz Struk, Herbert Xu, David S. Miller, keyrings,
linux-crypto, linux-kernel, Andrey Ryabinin, stable
We shouldn't free cert->pub->key in x509_cert_parse() because
x509_free_certificate() also does this:
BUG: Double free or freeing an invalid pointer
...
Call Trace:
[<ffffffff81896c20>] dump_stack+0x63/0x83
[<ffffffff81356571>] kasan_object_err+0x21/0x70
[<ffffffff81356ed9>] kasan_report_double_free+0x49/0x60
[<ffffffff813561ad>] kasan_slab_free+0x9d/0xc0
[<ffffffff81350b7a>] kfree+0x8a/0x1a0
[<ffffffff81844fbf>] public_key_free+0x1f/0x30
[<ffffffff818455d4>] x509_free_certificate+0x24/0x90
[<ffffffff818460bc>] x509_cert_parse+0x2bc/0x300
[<ffffffff81846cae>] x509_key_preparse+0x3e/0x330
[<ffffffff818444cf>] asymmetric_key_preparse+0x6f/0x100
[<ffffffff8178bec0>] key_create_or_update+0x260/0x5f0
[<ffffffff8178e6d9>] SyS_add_key+0x199/0x2a0
[<ffffffff821d823b>] entry_SYSCALL_64_fastpath+0x1e/0xad
Object at ffff880110bd1900, in cache kmalloc-512 size: 512
....
Freed:
PID = 2579
[<ffffffff8104283b>] save_stack_trace+0x1b/0x20
[<ffffffff813558f6>] save_stack+0x46/0xd0
[<ffffffff81356183>] kasan_slab_free+0x73/0xc0
[<ffffffff81350b7a>] kfree+0x8a/0x1a0
[<ffffffff818460a3>] x509_cert_parse+0x2a3/0x300
[<ffffffff81846cae>] x509_key_preparse+0x3e/0x330
[<ffffffff818444cf>] asymmetric_key_preparse+0x6f/0x100
[<ffffffff8178bec0>] key_create_or_update+0x260/0x5f0
[<ffffffff8178e6d9>] SyS_add_key+0x199/0x2a0
[<ffffffff821d823b>] entry_SYSCALL_64_fastpath+0x1e/0xad
Fixes: db6c43bd2132 ("crypto: KEYS: convert public key and digsig asym to the akcipher api")
Signed-off-by: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: <stable@vger.kernel.org>
---
crypto/asymmetric_keys/x509_cert_parser.c | 1 -
1 file changed, 1 deletion(-)
diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c
index 865f46e..c80765b 100644
--- a/crypto/asymmetric_keys/x509_cert_parser.c
+++ b/crypto/asymmetric_keys/x509_cert_parser.c
@@ -133,7 +133,6 @@ struct x509_certificate *x509_cert_parse(const void *data, size_t datalen)
return cert;
error_decode:
- kfree(cert->pub->key);
kfree(ctx);
error_no_ctx:
x509_free_certificate(cert);
--
2.7.3
^ permalink raw reply related
* Crypto Fixes for 4.9
From: Herbert Xu @ 2016-11-23 5:36 UTC (permalink / raw)
To: Linus Torvalds, David S. Miller, Linux Kernel Mailing List,
Linux Crypto Mailing List
In-Reply-To: <20161119102748.GA4277@gondor.apana.org.au>
Hi Linus:
The last push broke algif_hash for all shash implementations,
so this is a follow-up to fix that. It also fixes a problem
in the crypto scatterwalk that triggers a BUG_ON with certain
debugging options due to the new vmalloced-stack code.
Please pull from
git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git linus
Herbert Xu (2):
crypto: algif_hash - Fix result clobbering in recvmsg
crypto: scatterwalk - Remove unnecessary aliasing check in map_and_copy
crypto/algif_hash.c | 2 +-
crypto/scatterwalk.c | 4 ----
2 files changed, 1 insertion(+), 5 deletions(-)
Thanks,
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply
* Re: [PATCH] crypto: powerpc - Rename CRYPT_CRC32C_VPMSUM option
From: Anton Blanchard @ 2016-11-22 22:23 UTC (permalink / raw)
To: Jean Delvare
Cc: linux-crypto, Herbert Xu, Benjamin Herrenschmidt, Paul Mackerras,
Michael Ellerman, David S. Miller
In-Reply-To: <20161122103244.38d435cd@endymion>
Hi Jean,
> For consistency with the other 246 kernel configuration options,
> rename CRYPT_CRC32C_VPMSUM to CRYPTO_CRC32C_VPMSUM.
Thanks! Not sure how I missed that.
Acked-by: Anton Blanchard <anton@samba.org>
Anton
^ permalink raw reply
* [PATCH 06/10] crypto: caam - remove unneded dependencies on CRYPTO_DEV_FSL_CAAM
From: Horia Geantă @ 2016-11-22 13:44 UTC (permalink / raw)
To: Herbert Xu
Cc: David S. Miller, linux-crypto, Dan Douglass, Tudor Ambarus,
Alexandru Porosanu
In-Reply-To: <1479822252-23833-1-git-send-email-horia.geanta@nxp.com>
Remove dependency on CRYPTO_DEV_FSL_CAAM where superfluous:
depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR
is equivalent to
depends on CRYPTO_DEV_FSL_CAAM_JR
since CRYPTO_DEV_FSL_CAAM_JR depends on CRYPTO_DEV_FSL_CAAM.
Signed-off-by: Horia Geantă <horia.geanta@nxp.com>
---
drivers/crypto/caam/Kconfig | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig
index 64bf3024b680..ebeada75ab2d 100644
--- a/drivers/crypto/caam/Kconfig
+++ b/drivers/crypto/caam/Kconfig
@@ -74,7 +74,7 @@ config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD
config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
tristate "Register algorithm implementations with the Crypto API"
- depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR
+ depends on CRYPTO_DEV_FSL_CAAM_JR
default y
select CRYPTO_AEAD
select CRYPTO_AUTHENC
@@ -89,7 +89,7 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
config CRYPTO_DEV_FSL_CAAM_AHASH_API
tristate "Register hash algorithm implementations with Crypto API"
- depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR
+ depends on CRYPTO_DEV_FSL_CAAM_JR
default y
select CRYPTO_HASH
help
@@ -101,7 +101,7 @@ config CRYPTO_DEV_FSL_CAAM_AHASH_API
config CRYPTO_DEV_FSL_CAAM_PKC_API
tristate "Register public key cryptography implementations with Crypto API"
- depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR
+ depends on CRYPTO_DEV_FSL_CAAM_JR
default y
select CRYPTO_RSA
help
@@ -113,7 +113,7 @@ config CRYPTO_DEV_FSL_CAAM_PKC_API
config CRYPTO_DEV_FSL_CAAM_RNG_API
tristate "Register caam device for hwrng API"
- depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR
+ depends on CRYPTO_DEV_FSL_CAAM_JR
default y
select CRYPTO_RNG
select HW_RANDOM
--
2.4.4
^ permalink raw reply related
* [PATCH 08/10] crypto: caam - consolidate split key length computation
From: Horia Geantă @ 2016-11-22 13:44 UTC (permalink / raw)
To: Herbert Xu
Cc: David S. Miller, linux-crypto, Dan Douglass, Tudor Ambarus,
Alexandru Porosanu
In-Reply-To: <1479822252-23833-1-git-send-email-horia.geanta@nxp.com>
Move split key length and padded length computation from caamalg.c
and caamhash.c to key_gen.c.
Signed-off-by: Horia Geantă <horia.geanta@nxp.com>
---
drivers/crypto/caam/caamalg.c | 24 +++------------------
drivers/crypto/caam/caamhash.c | 24 ++-------------------
drivers/crypto/caam/key_gen.c | 47 +++++++++++++++++++++++++++++++++++++++++-
drivers/crypto/caam/key_gen.h | 3 ++-
4 files changed, 53 insertions(+), 45 deletions(-)
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 78b0b7c17205..767ffaea9649 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -586,18 +586,9 @@ static int rfc4543_setauthsize(struct crypto_aead *authenc,
return 0;
}
-static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in,
- u32 authkeylen)
-{
- return gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key_in,
- authkeylen);
-}
-
static int aead_setkey(struct crypto_aead *aead,
const u8 *key, unsigned int keylen)
{
- /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
- static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *jrdev = ctx->jrdev;
struct crypto_authenc_keys keys;
@@ -606,26 +597,17 @@ static int aead_setkey(struct crypto_aead *aead,
if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
goto badkey;
- /* Pick class 2 key length from algorithm submask */
- ctx->adata.keylen = mdpadlen[(ctx->adata.algtype &
- OP_ALG_ALGSEL_SUBMASK) >>
- OP_ALG_ALGSEL_SHIFT] * 2;
- ctx->adata.keylen_pad = ALIGN(ctx->adata.keylen, 16);
-
- if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
- goto badkey;
-
#ifdef DEBUG
printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
keys.authkeylen + keys.enckeylen, keys.enckeylen,
keys.authkeylen);
- printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
- ctx->adata.keylen, ctx->adata.keylen_pad);
print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
- ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen);
+ ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey,
+ keys.authkeylen, CAAM_MAX_KEY_SIZE -
+ keys.enckeylen);
if (ret) {
goto badkey;
}
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index d3f0ae16a73b..505007d0277c 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -398,12 +398,6 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
return 0;
}
-static int gen_split_hash_key(struct caam_hash_ctx *ctx, const u8 *key_in,
- u32 keylen)
-{
- return gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key_in, keylen);
-}
-
/* Digest hash size if it is too large */
static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
u32 *keylen, u8 *key_out, u32 digestsize)
@@ -483,8 +477,6 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
static int ahash_setkey(struct crypto_ahash *ahash,
const u8 *key, unsigned int keylen)
{
- /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
- static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
struct device *jrdev = ctx->jrdev;
int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
@@ -509,20 +501,8 @@ static int ahash_setkey(struct crypto_ahash *ahash,
key = hashed_key;
}
- /* Pick class 2 key length from algorithm submask */
- ctx->adata.keylen = mdpadlen[(ctx->adata.algtype &
- OP_ALG_ALGSEL_SUBMASK) >>
- OP_ALG_ALGSEL_SHIFT] * 2;
- ctx->adata.keylen_pad = ALIGN(ctx->adata.keylen, 16);
-
-#ifdef DEBUG
- printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
- ctx->adata.keylen, ctx->adata.keylen_pad);
- print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
-
- ret = gen_split_hash_key(ctx, key, keylen);
+ ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, keylen,
+ CAAM_MAX_HASH_KEY_SIZE);
if (ret)
goto bad_free_key;
diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c
index 621199a02f2e..1bb2816a9b4d 100644
--- a/drivers/crypto/caam/key_gen.c
+++ b/drivers/crypto/caam/key_gen.c
@@ -10,6 +10,36 @@
#include "desc_constr.h"
#include "key_gen.h"
+/**
+ * split_key_len - Compute MDHA split key length for a given algorithm
+ * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1,
+ * SHA224, SHA384, SHA512.
+ *
+ * Return: MDHA split key length
+ */
+static inline u32 split_key_len(u32 hash)
+{
+ /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
+ static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
+ u32 idx;
+
+ idx = (hash & OP_ALG_ALGSEL_SUBMASK) >> OP_ALG_ALGSEL_SHIFT;
+
+ return (u32)(mdpadlen[idx] * 2);
+}
+
+/**
+ * split_key_pad_len - Compute MDHA split key pad length for a given algorithm
+ * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1,
+ * SHA224, SHA384, SHA512.
+ *
+ * Return: MDHA split key pad length
+ */
+static inline u32 split_key_pad_len(u32 hash)
+{
+ return ALIGN(split_key_len(hash), 16);
+}
+
void split_key_done(struct device *dev, u32 *desc, u32 err,
void *context)
{
@@ -42,13 +72,28 @@ Split key generation-----------------------------------------------
@0xffe04000
*/
int gen_split_key(struct device *jrdev, u8 *key_out,
- struct alginfo * const adata, const u8 *key_in, u32 keylen)
+ struct alginfo * const adata, const u8 *key_in, u32 keylen,
+ int max_keylen)
{
u32 *desc;
struct split_key_result result;
dma_addr_t dma_addr_in, dma_addr_out;
int ret = -ENOMEM;
+ adata->keylen = split_key_len(adata->algtype & OP_ALG_ALGSEL_MASK);
+ adata->keylen_pad = split_key_pad_len(adata->algtype &
+ OP_ALG_ALGSEL_MASK);
+
+#ifdef DEBUG
+ dev_err(jrdev, "split keylen %d split keylen padded %d\n",
+ adata->keylen, adata->keylen_pad);
+ print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1);
+#endif
+
+ if (adata->keylen_pad > max_keylen)
+ return -EINVAL;
+
desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
if (!desc) {
dev_err(jrdev, "unable to allocate key input memory\n");
diff --git a/drivers/crypto/caam/key_gen.h b/drivers/crypto/caam/key_gen.h
index e87483c6057b..4628f389eb64 100644
--- a/drivers/crypto/caam/key_gen.h
+++ b/drivers/crypto/caam/key_gen.h
@@ -13,4 +13,5 @@ struct split_key_result {
void split_key_done(struct device *dev, u32 *desc, u32 err, void *context);
int gen_split_key(struct device *jrdev, u8 *key_out,
- struct alginfo * const adata, const u8 *key_in, u32 keylen);
+ struct alginfo * const adata, const u8 *key_in, u32 keylen,
+ int max_keylen);
--
2.4.4
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox