* [PATCH v2 3/7] crypto: arm64/aes-blk - expose AES-CTR as synchronous cipher as well
From: Ard Biesheuvel @ 2017-01-11 16:41 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1484152915-26517-1-git-send-email-ard.biesheuvel@linaro.org>
In addition to wrapping the AES-CTR cipher into the async SIMD wrapper,
which exposes it as an async skcipher that defers processing to process
context, expose our AES-CTR implementation directly as a synchronous cipher
as well, but with a lower priority.
This makes the AES-CTR transform usable in places where synchronous
transforms are required, such as the MAC802.11 encryption code, which
executes in sotfirq context, where SIMD processing is allowed on arm64.
Users of the async transform will keep the existing behavior.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/aes-glue.c | 25 ++++++++++++++++++--
1 file changed, 23 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index 4e3f8adb1793..5164aaf82c6a 100644
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -327,6 +327,23 @@ static struct skcipher_alg aes_algs[] = { {
.decrypt = ctr_encrypt,
}, {
.base = {
+ .cra_name = "ctr(aes)",
+ .cra_driver_name = "ctr-aes-" MODE,
+ .cra_priority = PRIO - 1,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct crypto_aes_ctx),
+ .cra_alignmask = 7,
+ .cra_module = THIS_MODULE,
+ },
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .chunksize = AES_BLOCK_SIZE,
+ .setkey = skcipher_aes_setkey,
+ .encrypt = ctr_encrypt,
+ .decrypt = ctr_encrypt,
+}, {
+ .base = {
.cra_name = "__xts(aes)",
.cra_driver_name = "__xts-aes-" MODE,
.cra_priority = PRIO,
@@ -350,8 +367,9 @@ static void aes_exit(void)
{
int i;
- for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++)
- simd_skcipher_free(aes_simd_algs[i]);
+ for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
+ if (aes_simd_algs[i])
+ simd_skcipher_free(aes_simd_algs[i]);
crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
}
@@ -370,6 +388,9 @@ static int __init aes_init(void)
return err;
for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
+ if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
+ continue;
+
algname = aes_algs[i].base.cra_name + 2;
drvname = aes_algs[i].base.cra_driver_name + 2;
basename = aes_algs[i].base.cra_driver_name;
--
2.7.4
^ permalink raw reply related
* [PATCH v2 4/7] crypto: arm64/aes - add scalar implementation
From: Ard Biesheuvel @ 2017-01-11 16:41 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1484152915-26517-1-git-send-email-ard.biesheuvel@linaro.org>
This adds a scalar implementation of AES, based on the precomputed tables
that are exposed by the generic AES code. Since rotates are cheap on arm64,
this implementation only uses the 4 core tables (of 1 KB each), and avoids
the prerotated ones, reducing the D-cache footprint by 75%.
On Cortex-A57, this code manages 13.0 cycles per byte, which is ~34% faster
than the generic C code. (Note that this is still >13x slower than the code
that uses the optional ARMv8 Crypto Extensions, which manages <1 cycles per
byte.)
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/Kconfig | 4 +
arch/arm64/crypto/Makefile | 3 +
arch/arm64/crypto/aes-cipher-core.S | 127 ++++++++++++++++++++
arch/arm64/crypto/aes-cipher-glue.c | 69 +++++++++++
4 files changed, 203 insertions(+)
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 0bf0f531f539..0826f8e599a6 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -41,6 +41,10 @@ config CRYPTO_CRC32_ARM64_CE
depends on KERNEL_MODE_NEON && CRC32
select CRYPTO_HASH
+config CRYPTO_AES_ARM64
+ tristate "AES core cipher using scalar instructions"
+ select CRYPTO_AES
+
config CRYPTO_AES_ARM64_CE
tristate "AES core cipher using ARMv8 Crypto Extensions"
depends on ARM64 && KERNEL_MODE_NEON
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index 9d2826c5fccf..a893507629eb 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -44,6 +44,9 @@ sha512-arm64-y := sha512-glue.o sha512-core.o
obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha20-neon.o
chacha20-neon-y := chacha20-neon-core.o chacha20-neon-glue.o
+obj-$(CONFIG_CRYPTO_AES_ARM64) += aes-arm64.o
+aes-arm64-y := aes-cipher-core.o aes-cipher-glue.o
+
AFLAGS_aes-ce.o := -DINTERLEAVE=4
AFLAGS_aes-neon.o := -DINTERLEAVE=4
diff --git a/arch/arm64/crypto/aes-cipher-core.S b/arch/arm64/crypto/aes-cipher-core.S
new file mode 100644
index 000000000000..37590ab8121a
--- /dev/null
+++ b/arch/arm64/crypto/aes-cipher-core.S
@@ -0,0 +1,127 @@
+/*
+ * Scalar AES core transform
+ *
+ * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+ .text
+
+ rk .req x0
+ out .req x1
+ in .req x2
+ rounds .req x3
+ tt .req x4
+ lt .req x2
+
+ .macro __hround, out0, out1, in0, in1, in2, in3, t0, t1, enc
+ ldp \out0, \out1, [rk], #8
+
+ ubfx w13, \in0, #0, #8
+ ubfx w14, \in1, #8, #8
+ ldr w13, [tt, w13, uxtw #2]
+ ldr w14, [tt, w14, uxtw #2]
+
+ .if \enc
+ ubfx w17, \in1, #0, #8
+ ubfx w18, \in2, #8, #8
+ .else
+ ubfx w17, \in3, #0, #8
+ ubfx w18, \in0, #8, #8
+ .endif
+ ldr w17, [tt, w17, uxtw #2]
+ ldr w18, [tt, w18, uxtw #2]
+
+ ubfx w15, \in2, #16, #8
+ ubfx w16, \in3, #24, #8
+ ldr w15, [tt, w15, uxtw #2]
+ ldr w16, [tt, w16, uxtw #2]
+
+ .if \enc
+ ubfx \t0, \in3, #16, #8
+ ubfx \t1, \in0, #24, #8
+ .else
+ ubfx \t0, \in1, #16, #8
+ ubfx \t1, \in2, #24, #8
+ .endif
+ ldr \t0, [tt, \t0, uxtw #2]
+ ldr \t1, [tt, \t1, uxtw #2]
+
+ eor \out0, \out0, w13
+ eor \out1, \out1, w17
+ eor \out0, \out0, w14, ror #24
+ eor \out1, \out1, w18, ror #24
+ eor \out0, \out0, w15, ror #16
+ eor \out1, \out1, \t0, ror #16
+ eor \out0, \out0, w16, ror #8
+ eor \out1, \out1, \t1, ror #8
+ .endm
+
+ .macro fround, out0, out1, out2, out3, in0, in1, in2, in3
+ __hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1
+ __hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1
+ .endm
+
+ .macro iround, out0, out1, out2, out3, in0, in1, in2, in3
+ __hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0
+ __hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0
+ .endm
+
+ .macro do_crypt, round, ttab, ltab
+ ldp w5, w6, [in]
+ ldp w7, w8, [in, #8]
+ ldp w9, w10, [rk], #16
+ ldp w11, w12, [rk, #-8]
+
+CPU_BE( rev w5, w5 )
+CPU_BE( rev w6, w6 )
+CPU_BE( rev w7, w7 )
+CPU_BE( rev w8, w8 )
+
+ eor w5, w5, w9
+ eor w6, w6, w10
+ eor w7, w7, w11
+ eor w8, w8, w12
+
+ ldr tt, =\ttab
+ ldr lt, =\ltab
+
+ tbnz rounds, #1, 1f
+
+0: \round w9, w10, w11, w12, w5, w6, w7, w8
+ \round w5, w6, w7, w8, w9, w10, w11, w12
+
+1: subs rounds, rounds, #4
+ \round w9, w10, w11, w12, w5, w6, w7, w8
+ csel tt, tt, lt, hi
+ \round w5, w6, w7, w8, w9, w10, w11, w12
+ b.hi 0b
+
+CPU_BE( rev w5, w5 )
+CPU_BE( rev w6, w6 )
+CPU_BE( rev w7, w7 )
+CPU_BE( rev w8, w8 )
+
+ stp w5, w6, [out]
+ stp w7, w8, [out, #8]
+ ret
+
+ .align 4
+ .ltorg
+ .endm
+
+ .align 5
+ENTRY(__aes_arm64_encrypt)
+ do_crypt fround, crypto_ft_tab, crypto_fl_tab
+ENDPROC(__aes_arm64_encrypt)
+
+ .align 5
+ENTRY(__aes_arm64_decrypt)
+ do_crypt iround, crypto_it_tab, crypto_il_tab
+ENDPROC(__aes_arm64_decrypt)
diff --git a/arch/arm64/crypto/aes-cipher-glue.c b/arch/arm64/crypto/aes-cipher-glue.c
new file mode 100644
index 000000000000..7288e7cbebff
--- /dev/null
+++ b/arch/arm64/crypto/aes-cipher-glue.c
@@ -0,0 +1,69 @@
+/*
+ * Scalar AES core transform
+ *
+ * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <crypto/aes.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+
+asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
+EXPORT_SYMBOL(__aes_arm64_encrypt);
+
+asmlinkage void __aes_arm64_decrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
+EXPORT_SYMBOL(__aes_arm64_decrypt);
+
+static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ int rounds = 6 + ctx->key_length / 4;
+
+ __aes_arm64_encrypt(ctx->key_enc, out, in, rounds);
+}
+
+static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ int rounds = 6 + ctx->key_length / 4;
+
+ __aes_arm64_decrypt(ctx->key_dec, out, in, rounds);
+}
+
+static struct crypto_alg aes_alg = {
+ .cra_name = "aes",
+ .cra_driver_name = "aes-arm64",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct crypto_aes_ctx),
+ .cra_module = THIS_MODULE,
+
+ .cra_cipher.cia_min_keysize = AES_MIN_KEY_SIZE,
+ .cra_cipher.cia_max_keysize = AES_MAX_KEY_SIZE,
+ .cra_cipher.cia_setkey = crypto_aes_set_key,
+ .cra_cipher.cia_encrypt = aes_encrypt,
+ .cra_cipher.cia_decrypt = aes_decrypt
+};
+
+static int __init aes_init(void)
+{
+ return crypto_register_alg(&aes_alg);
+}
+
+static void __exit aes_fini(void)
+{
+ crypto_unregister_alg(&aes_alg);
+}
+
+module_init(aes_init);
+module_exit(aes_fini);
+
+MODULE_DESCRIPTION("Scalar AES cipher for arm64");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("aes");
--
2.7.4
^ permalink raw reply related
* [PATCH v2 5/7] crypto: arm/aes - replace scalar AES cipher
From: Ard Biesheuvel @ 2017-01-11 16:41 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1484152915-26517-1-git-send-email-ard.biesheuvel@linaro.org>
This replaces the scalar AES cipher that originates in the OpenSSL project
with a new implementation that is ~15% (*) faster (on modern cores), and
reuses the lookup tables and the key schedule generation routines from the
generic C implementation (which is usually compiled in anyway due to
networking and other subsystems depending on it).
Note that the bit sliced NEON code for AES still depends on the scalar cipher
that this patch replaces, so it is not removed entirely yet.
* On Cortex-A57, the performance increases from 17.0 to 14.9 cycles per byte
for 128-bit keys.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm/crypto/Kconfig | 20 +--
arch/arm/crypto/Makefile | 4 +-
arch/arm/crypto/aes-cipher-core.S | 179 ++++++++++++++++++++
arch/arm/crypto/aes-cipher-glue.c | 74 ++++++++
arch/arm/crypto/aes_glue.c | 98 -----------
5 files changed, 256 insertions(+), 119 deletions(-)
diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig
index 2f3339f015d3..f1de658c3c8f 100644
--- a/arch/arm/crypto/Kconfig
+++ b/arch/arm/crypto/Kconfig
@@ -62,33 +62,15 @@ config CRYPTO_SHA512_ARM
using optimized ARM assembler and NEON, when available.
config CRYPTO_AES_ARM
- tristate "AES cipher algorithms (ARM-asm)"
- depends on ARM
+ tristate "Scalar AES cipher for ARM"
select CRYPTO_ALGAPI
select CRYPTO_AES
help
Use optimized AES assembler routines for ARM platforms.
- AES cipher algorithms (FIPS-197). AES uses the Rijndael
- algorithm.
-
- Rijndael appears to be consistently a very good performer in
- both hardware and software across a wide range of computing
- environments regardless of its use in feedback or non-feedback
- modes. Its key setup time is excellent, and its key agility is
- good. Rijndael's very low memory requirements make it very well
- suited for restricted-space environments, in which it also
- demonstrates excellent performance. Rijndael's operations are
- among the easiest to defend against power and timing attacks.
-
- The AES specifies three key sizes: 128, 192 and 256 bits
-
- See <http://csrc.nist.gov/encryption/aes/> for more information.
-
config CRYPTO_AES_ARM_BS
tristate "Bit sliced AES using NEON instructions"
depends on KERNEL_MODE_NEON
- select CRYPTO_AES_ARM
select CRYPTO_BLKCIPHER
select CRYPTO_SIMD
help
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
index 8d74e55eacd4..8f5de2db701c 100644
--- a/arch/arm/crypto/Makefile
+++ b/arch/arm/crypto/Makefile
@@ -27,8 +27,8 @@ $(warning $(ce-obj-y) $(ce-obj-m))
endif
endif
-aes-arm-y := aes-armv4.o aes_glue.o
-aes-arm-bs-y := aesbs-core.o aesbs-glue.o
+aes-arm-y := aes-cipher-core.o aes-cipher-glue.o
+aes-arm-bs-y := aes-armv4.o aesbs-core.o aesbs-glue.o
sha1-arm-y := sha1-armv4-large.o sha1_glue.o
sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o
sha256-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha256_neon_glue.o
diff --git a/arch/arm/crypto/aes-cipher-core.S b/arch/arm/crypto/aes-cipher-core.S
new file mode 100644
index 000000000000..b04261e1e068
--- /dev/null
+++ b/arch/arm/crypto/aes-cipher-core.S
@@ -0,0 +1,179 @@
+/*
+ * Scalar AES core transform
+ *
+ * Copyright (C) 2017 Linaro Ltd.
+ * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+
+ .text
+ .align 5
+
+ rk .req r0
+ rounds .req r1
+ in .req r2
+ out .req r3
+ tt .req ip
+
+ t0 .req lr
+ t1 .req r2
+ t2 .req r3
+
+ .macro __select, out, in, idx
+ .if __LINUX_ARM_ARCH__ < 7
+ and \out, \in, #0xff << (8 * \idx)
+ .else
+ ubfx \out, \in, #(8 * \idx), #8
+ .endif
+ .endm
+
+ .macro __load, out, in, idx
+ .if __LINUX_ARM_ARCH__ < 7 && \idx > 0
+ ldr \out, [tt, \in, lsr #(8 * \idx) - 2]
+ .else
+ ldr \out, [tt, \in, lsl #2]
+ .endif
+ .endm
+
+ .macro __hround, out0, out1, in0, in1, in2, in3, t3, t4, enc
+ __select \out0, \in0, 0
+ __select t0, \in1, 1
+ __load \out0, \out0, 0
+ __load t0, t0, 1
+
+ .if \enc
+ __select \out1, \in1, 0
+ __select t1, \in2, 1
+ .else
+ __select \out1, \in3, 0
+ __select t1, \in0, 1
+ .endif
+ __load \out1, \out1, 0
+ __select t2, \in2, 2
+ __load t1, t1, 1
+ __load t2, t2, 2
+
+ eor \out0, \out0, t0, ror #24
+
+ __select t0, \in3, 3
+ .if \enc
+ __select \t3, \in3, 2
+ __select \t4, \in0, 3
+ .else
+ __select \t3, \in1, 2
+ __select \t4, \in2, 3
+ .endif
+ __load \t3, \t3, 2
+ __load t0, t0, 3
+ __load \t4, \t4, 3
+
+ eor \out1, \out1, t1, ror #24
+ eor \out0, \out0, t2, ror #16
+ ldm rk!, {t1, t2}
+ eor \out1, \out1, \t3, ror #16
+ eor \out0, \out0, t0, ror #8
+ eor \out1, \out1, \t4, ror #8
+ eor \out0, \out0, t1
+ eor \out1, \out1, t2
+ .endm
+
+ .macro fround, out0, out1, out2, out3, in0, in1, in2, in3
+ __hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1
+ __hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1
+ .endm
+
+ .macro iround, out0, out1, out2, out3, in0, in1, in2, in3
+ __hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0
+ __hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0
+ .endm
+
+ .macro __rev, out, in
+ .if __LINUX_ARM_ARCH__ < 6
+ lsl t0, \in, #24
+ and t1, \in, #0xff00
+ and t2, \in, #0xff0000
+ orr \out, t0, \in, lsr #24
+ orr \out, \out, t1, lsl #8
+ orr \out, \out, t2, lsr #8
+ .else
+ rev \out, \in
+ .endif
+ .endm
+
+ .macro __adrl, out, sym, c
+ .if __LINUX_ARM_ARCH__ < 7
+ ldr\c \out, =\sym
+ .else
+ movw\c \out, #:lower16:\sym
+ movt\c \out, #:upper16:\sym
+ .endif
+ .endm
+
+ .macro do_crypt, round, ttab, ltab
+ push {r3-r11, lr}
+
+ ldr r4, [in]
+ ldr r5, [in, #4]
+ ldr r6, [in, #8]
+ ldr r7, [in, #12]
+
+ ldm rk!, {r8-r11}
+
+#ifdef CONFIG_CPU_BIG_ENDIAN
+ __rev r4, r4
+ __rev r5, r5
+ __rev r6, r6
+ __rev r7, r7
+#endif
+
+ eor r4, r4, r8
+ eor r5, r5, r9
+ eor r6, r6, r10
+ eor r7, r7, r11
+
+ __adrl tt, \ttab
+
+ tst rounds, #2
+ bne 1f
+
+0: \round r8, r9, r10, r11, r4, r5, r6, r7
+ \round r4, r5, r6, r7, r8, r9, r10, r11
+
+1: subs rounds, rounds, #4
+ \round r8, r9, r10, r11, r4, r5, r6, r7
+ __adrl tt, \ltab, ls
+ \round r4, r5, r6, r7, r8, r9, r10, r11
+ bhi 0b
+
+#ifdef CONFIG_CPU_BIG_ENDIAN
+ __rev r4, r4
+ __rev r5, r5
+ __rev r6, r6
+ __rev r7, r7
+#endif
+
+ ldr out, [sp]
+
+ str r4, [out]
+ str r5, [out, #4]
+ str r6, [out, #8]
+ str r7, [out, #12]
+
+ pop {r3-r11, pc}
+
+ .align 3
+ .ltorg
+ .endm
+
+ENTRY(__aes_arm_encrypt)
+ do_crypt fround, crypto_ft_tab, crypto_fl_tab
+ENDPROC(__aes_arm_encrypt)
+
+ENTRY(__aes_arm_decrypt)
+ do_crypt iround, crypto_it_tab, crypto_il_tab
+ENDPROC(__aes_arm_decrypt)
diff --git a/arch/arm/crypto/aes-cipher-glue.c b/arch/arm/crypto/aes-cipher-glue.c
new file mode 100644
index 000000000000..c222f6e072ad
--- /dev/null
+++ b/arch/arm/crypto/aes-cipher-glue.c
@@ -0,0 +1,74 @@
+/*
+ * Scalar AES core transform
+ *
+ * Copyright (C) 2017 Linaro Ltd.
+ * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <crypto/aes.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+
+asmlinkage void __aes_arm_encrypt(u32 *rk, int rounds, const u8 *in, u8 *out);
+EXPORT_SYMBOL(__aes_arm_encrypt);
+
+asmlinkage void __aes_arm_decrypt(u32 *rk, int rounds, const u8 *in, u8 *out);
+EXPORT_SYMBOL(__aes_arm_decrypt);
+
+static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ int rounds = 6 + ctx->key_length / 4;
+
+ __aes_arm_encrypt(ctx->key_enc, rounds, in, out);
+}
+
+static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ int rounds = 6 + ctx->key_length / 4;
+
+ __aes_arm_decrypt(ctx->key_dec, rounds, in, out);
+}
+
+static struct crypto_alg aes_alg = {
+ .cra_name = "aes",
+ .cra_driver_name = "aes-arm",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct crypto_aes_ctx),
+ .cra_module = THIS_MODULE,
+
+ .cra_cipher.cia_min_keysize = AES_MIN_KEY_SIZE,
+ .cra_cipher.cia_max_keysize = AES_MAX_KEY_SIZE,
+ .cra_cipher.cia_setkey = crypto_aes_set_key,
+ .cra_cipher.cia_encrypt = aes_encrypt,
+ .cra_cipher.cia_decrypt = aes_decrypt,
+
+#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+ .cra_alignmask = 3,
+#endif
+};
+
+static int __init aes_init(void)
+{
+ return crypto_register_alg(&aes_alg);
+}
+
+static void __exit aes_fini(void)
+{
+ crypto_unregister_alg(&aes_alg);
+}
+
+module_init(aes_init);
+module_exit(aes_fini);
+
+MODULE_DESCRIPTION("Scalar AES cipher for ARM");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("aes");
diff --git a/arch/arm/crypto/aes_glue.c b/arch/arm/crypto/aes_glue.c
deleted file mode 100644
index 0409b8f89782..000000000000
--- a/arch/arm/crypto/aes_glue.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Glue Code for the asm optimized version of the AES Cipher Algorithm
- */
-
-#include <linux/module.h>
-#include <linux/crypto.h>
-#include <crypto/aes.h>
-
-#include "aes_glue.h"
-
-EXPORT_SYMBOL(AES_encrypt);
-EXPORT_SYMBOL(AES_decrypt);
-EXPORT_SYMBOL(private_AES_set_encrypt_key);
-EXPORT_SYMBOL(private_AES_set_decrypt_key);
-
-static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
- struct AES_CTX *ctx = crypto_tfm_ctx(tfm);
- AES_encrypt(src, dst, &ctx->enc_key);
-}
-
-static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
- struct AES_CTX *ctx = crypto_tfm_ctx(tfm);
- AES_decrypt(src, dst, &ctx->dec_key);
-}
-
-static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
- unsigned int key_len)
-{
- struct AES_CTX *ctx = crypto_tfm_ctx(tfm);
-
- switch (key_len) {
- case AES_KEYSIZE_128:
- key_len = 128;
- break;
- case AES_KEYSIZE_192:
- key_len = 192;
- break;
- case AES_KEYSIZE_256:
- key_len = 256;
- break;
- default:
- tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
- return -EINVAL;
- }
-
- if (private_AES_set_encrypt_key(in_key, key_len, &ctx->enc_key) == -1) {
- tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
- return -EINVAL;
- }
- /* private_AES_set_decrypt_key expects an encryption key as input */
- ctx->dec_key = ctx->enc_key;
- if (private_AES_set_decrypt_key(in_key, key_len, &ctx->dec_key) == -1) {
- tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
- return -EINVAL;
- }
- return 0;
-}
-
-static struct crypto_alg aes_alg = {
- .cra_name = "aes",
- .cra_driver_name = "aes-asm",
- .cra_priority = 200,
- .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct AES_CTX),
- .cra_module = THIS_MODULE,
- .cra_list = LIST_HEAD_INIT(aes_alg.cra_list),
- .cra_u = {
- .cipher = {
- .cia_min_keysize = AES_MIN_KEY_SIZE,
- .cia_max_keysize = AES_MAX_KEY_SIZE,
- .cia_setkey = aes_set_key,
- .cia_encrypt = aes_encrypt,
- .cia_decrypt = aes_decrypt
- }
- }
-};
-
-static int __init aes_init(void)
-{
- return crypto_register_alg(&aes_alg);
-}
-
-static void __exit aes_fini(void)
-{
- crypto_unregister_alg(&aes_alg);
-}
-
-module_init(aes_init);
-module_exit(aes_fini);
-
-MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm (ASM)");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS_CRYPTO("aes");
-MODULE_ALIAS_CRYPTO("aes-asm");
-MODULE_AUTHOR("David McCullough <ucdevel@gmail.com>");
--
2.7.4
^ permalink raw reply related
* [PATCH v2 7/7] crypto: arm64/aes - reimplement bit-sliced ARM/NEON implementation for arm64
From: Ard Biesheuvel @ 2017-01-11 16:41 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1484152915-26517-1-git-send-email-ard.biesheuvel@linaro.org>
This is a reimplementation of the NEON version of the bit-sliced AES
algorithm. This code is heavily based on Andy Polyakov's OpenSSL version
for ARM, which is also available in the kernel. This is an alternative for
the existing NEON implementation for arm64 authored by me, which suffers
from poor performance due to its reliance on the pathologically slow four
register variant of the tbl/tbx NEON instruction.
This version is about ~30% (*) faster than the generic C code, but only in
cases where the input can be 8x interleaved (this is a fundamental property
of bit slicing). For this reason, only the chaining modes ECB, XTS and CTR
are implemented. (The significance of ECB is that it could potentially be
used by other chaining modes)
* Measured on Cortex-A57. Note that this is still an order of magnitude
slower than the implementations that use the dedicated AES instructions
introduced in ARMv8, but those are part of an optional extension, and so
it is good to have a fallback.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/Kconfig | 7 +
arch/arm64/crypto/Makefile | 3 +
arch/arm64/crypto/aes-neonbs-core.S | 963 ++++++++++++++++++++
arch/arm64/crypto/aes-neonbs-glue.c | 420 +++++++++
4 files changed, 1393 insertions(+)
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 0826f8e599a6..5de75c3dcbd4 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -82,4 +82,11 @@ config CRYPTO_CHACHA20_NEON
select CRYPTO_BLKCIPHER
select CRYPTO_CHACHA20
+config CRYPTO_AES_ARM64_BS
+ tristate "AES in ECB/CBC/CTR/XTS modes using bit-sliced NEON algorithm"
+ depends on KERNEL_MODE_NEON
+ select CRYPTO_BLKCIPHER
+ select CRYPTO_AES_ARM64
+ select CRYPTO_SIMD
+
endif
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index a893507629eb..d1ae1b9cbe70 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -47,6 +47,9 @@ chacha20-neon-y := chacha20-neon-core.o chacha20-neon-glue.o
obj-$(CONFIG_CRYPTO_AES_ARM64) += aes-arm64.o
aes-arm64-y := aes-cipher-core.o aes-cipher-glue.o
+obj-$(CONFIG_CRYPTO_AES_ARM64_BS) += aes-neon-bs.o
+aes-neon-bs-y := aes-neonbs-core.o aes-neonbs-glue.o
+
AFLAGS_aes-ce.o := -DINTERLEAVE=4
AFLAGS_aes-neon.o := -DINTERLEAVE=4
diff --git a/arch/arm64/crypto/aes-neonbs-core.S b/arch/arm64/crypto/aes-neonbs-core.S
new file mode 100644
index 000000000000..8d0cdaa2768d
--- /dev/null
+++ b/arch/arm64/crypto/aes-neonbs-core.S
@@ -0,0 +1,963 @@
+/*
+ * Bit sliced AES using NEON instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/*
+ * The algorithm implemented here is described in detail by the paper
+ * 'Faster and Timing-Attack Resistant AES-GCM' by Emilia Kaesper and
+ * Peter Schwabe (https://eprint.iacr.org/2009/129.pdf)
+ *
+ * This implementation is based primarily on the OpenSSL implementation
+ * for 32-bit ARM written by Andy Polyakov <appro@openssl.org>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+ .text
+
+ rounds .req x11
+ bskey .req x12
+
+ .macro in_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
+ eor \b2, \b2, \b1
+ eor \b5, \b5, \b6
+ eor \b3, \b3, \b0
+ eor \b6, \b6, \b2
+ eor \b5, \b5, \b0
+ eor \b6, \b6, \b3
+ eor \b3, \b3, \b7
+ eor \b7, \b7, \b5
+ eor \b3, \b3, \b4
+ eor \b4, \b4, \b5
+ eor \b2, \b2, \b7
+ eor \b3, \b3, \b1
+ eor \b1, \b1, \b5
+ .endm
+
+ .macro out_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
+ eor \b0, \b0, \b6
+ eor \b1, \b1, \b4
+ eor \b4, \b4, \b6
+ eor \b2, \b2, \b0
+ eor \b6, \b6, \b1
+ eor \b1, \b1, \b5
+ eor \b5, \b5, \b3
+ eor \b3, \b3, \b7
+ eor \b7, \b7, \b5
+ eor \b2, \b2, \b5
+ eor \b4, \b4, \b7
+ .endm
+
+ .macro inv_in_bs_ch, b6, b1, b2, b4, b7, b0, b3, b5
+ eor \b1, \b1, \b7
+ eor \b4, \b4, \b7
+ eor \b7, \b7, \b5
+ eor \b1, \b1, \b3
+ eor \b2, \b2, \b5
+ eor \b3, \b3, \b7
+ eor \b6, \b6, \b1
+ eor \b2, \b2, \b0
+ eor \b5, \b5, \b3
+ eor \b4, \b4, \b6
+ eor \b0, \b0, \b6
+ eor \b1, \b1, \b4
+ .endm
+
+ .macro inv_out_bs_ch, b6, b5, b0, b3, b7, b1, b4, b2
+ eor \b1, \b1, \b5
+ eor \b2, \b2, \b7
+ eor \b3, \b3, \b1
+ eor \b4, \b4, \b5
+ eor \b7, \b7, \b5
+ eor \b3, \b3, \b4
+ eor \b5, \b5, \b0
+ eor \b3, \b3, \b7
+ eor \b6, \b6, \b2
+ eor \b2, \b2, \b1
+ eor \b6, \b6, \b3
+ eor \b3, \b3, \b0
+ eor \b5, \b5, \b6
+ .endm
+
+ .macro mul_gf4, x0, x1, y0, y1, t0, t1
+ eor \t0, \y0, \y1
+ and \t0, \t0, \x0
+ eor \x0, \x0, \x1
+ and \t1, \x1, \y0
+ and \x0, \x0, \y1
+ eor \x1, \t1, \t0
+ eor \x0, \x0, \t1
+ .endm
+
+ .macro mul_gf4_n_gf4, x0, x1, y0, y1, t0, x2, x3, y2, y3, t1
+ eor \t0, \y0, \y1
+ eor \t1, \y2, \y3
+ and \t0, \t0, \x0
+ and \t1, \t1, \x2
+ eor \x0, \x0, \x1
+ eor \x2, \x2, \x3
+ and \x1, \x1, \y0
+ and \x3, \x3, \y2
+ and \x0, \x0, \y1
+ and \x2, \x2, \y3
+ eor \x1, \x1, \x0
+ eor \x2, \x2, \x3
+ eor \x0, \x0, \t0
+ eor \x3, \x3, \t1
+ .endm
+
+ .macro mul_gf16_2, x0, x1, x2, x3, x4, x5, x6, x7, \
+ y0, y1, y2, y3, t0, t1, t2, t3
+ eor \t0, \x0, \x2
+ eor \t1, \x1, \x3
+ mul_gf4 \x0, \x1, \y0, \y1, \t2, \t3
+ eor \y0, \y0, \y2
+ eor \y1, \y1, \y3
+ mul_gf4_n_gf4 \t0, \t1, \y0, \y1, \t3, \x2, \x3, \y2, \y3, \t2
+ eor \x0, \x0, \t0
+ eor \x2, \x2, \t0
+ eor \x1, \x1, \t1
+ eor \x3, \x3, \t1
+ eor \t0, \x4, \x6
+ eor \t1, \x5, \x7
+ mul_gf4_n_gf4 \t0, \t1, \y0, \y1, \t3, \x6, \x7, \y2, \y3, \t2
+ eor \y0, \y0, \y2
+ eor \y1, \y1, \y3
+ mul_gf4 \x4, \x5, \y0, \y1, \t2, \t3
+ eor \x4, \x4, \t0
+ eor \x6, \x6, \t0
+ eor \x5, \x5, \t1
+ eor \x7, \x7, \t1
+ .endm
+
+ .macro inv_gf256, x0, x1, x2, x3, x4, x5, x6, x7, \
+ t0, t1, t2, t3, s0, s1, s2, s3
+ eor \t3, \x4, \x6
+ eor \t0, \x5, \x7
+ eor \t1, \x1, \x3
+ eor \s1, \x7, \x6
+ eor \s0, \x0, \x2
+ eor \s3, \t3, \t0
+ orr \t2, \t0, \t1
+ and \s2, \t3, \s0
+ orr \t3, \t3, \s0
+ eor \s0, \s0, \t1
+ and \t0, \t0, \t1
+ eor \t1, \x3, \x2
+ and \s3, \s3, \s0
+ and \s1, \s1, \t1
+ eor \t1, \x4, \x5
+ eor \s0, \x1, \x0
+ eor \t3, \t3, \s1
+ eor \t2, \t2, \s1
+ and \s1, \t1, \s0
+ orr \t1, \t1, \s0
+ eor \t3, \t3, \s3
+ eor \t0, \t0, \s1
+ eor \t2, \t2, \s2
+ eor \t1, \t1, \s3
+ eor \t0, \t0, \s2
+ and \s0, \x7, \x3
+ eor \t1, \t1, \s2
+ and \s1, \x6, \x2
+ and \s2, \x5, \x1
+ orr \s3, \x4, \x0
+ eor \t3, \t3, \s0
+ eor \t1, \t1, \s2
+ eor \s0, \t0, \s3
+ eor \t2, \t2, \s1
+ and \s2, \t3, \t1
+ eor \s1, \t2, \s2
+ eor \s3, \s0, \s2
+ bsl \s1, \t1, \s0
+ not \t0, \s0
+ bsl \s0, \s1, \s3
+ bsl \t0, \s1, \s3
+ bsl \s3, \t3, \t2
+ eor \t3, \t3, \t2
+ and \s2, \s0, \s3
+ eor \t1, \t1, \t0
+ eor \s2, \s2, \t3
+ mul_gf16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
+ \s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3
+ .endm
+
+ .macro sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
+ t0, t1, t2, t3, s0, s1, s2, s3
+ in_bs_ch \b0\().16b, \b1\().16b, \b2\().16b, \b3\().16b, \
+ \b4\().16b, \b5\().16b, \b6\().16b, \b7\().16b
+ inv_gf256 \b6\().16b, \b5\().16b, \b0\().16b, \b3\().16b, \
+ \b7\().16b, \b1\().16b, \b4\().16b, \b2\().16b, \
+ \t0\().16b, \t1\().16b, \t2\().16b, \t3\().16b, \
+ \s0\().16b, \s1\().16b, \s2\().16b, \s3\().16b
+ out_bs_ch \b7\().16b, \b1\().16b, \b4\().16b, \b2\().16b, \
+ \b6\().16b, \b5\().16b, \b0\().16b, \b3\().16b
+ .endm
+
+ .macro inv_sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
+ t0, t1, t2, t3, s0, s1, s2, s3
+ inv_in_bs_ch \b0\().16b, \b1\().16b, \b2\().16b, \b3\().16b, \
+ \b4\().16b, \b5\().16b, \b6\().16b, \b7\().16b
+ inv_gf256 \b5\().16b, \b1\().16b, \b2\().16b, \b6\().16b, \
+ \b3\().16b, \b7\().16b, \b0\().16b, \b4\().16b, \
+ \t0\().16b, \t1\().16b, \t2\().16b, \t3\().16b, \
+ \s0\().16b, \s1\().16b, \s2\().16b, \s3\().16b
+ inv_out_bs_ch \b3\().16b, \b7\().16b, \b0\().16b, \b4\().16b, \
+ \b5\().16b, \b1\().16b, \b2\().16b, \b6\().16b
+ .endm
+
+ .macro enc_next_rk
+ ldp q16, q17, [bskey], #128
+ ldp q18, q19, [bskey, #-96]
+ ldp q20, q21, [bskey, #-64]
+ ldp q22, q23, [bskey, #-32]
+ .endm
+
+ .macro dec_next_rk
+ ldp q16, q17, [bskey, #-128]!
+ ldp q18, q19, [bskey, #32]
+ ldp q20, q21, [bskey, #64]
+ ldp q22, q23, [bskey, #96]
+ .endm
+
+ .macro add_round_key, x0, x1, x2, x3, x4, x5, x6, x7
+ eor \x0\().16b, \x0\().16b, v16.16b
+ eor \x1\().16b, \x1\().16b, v17.16b
+ eor \x2\().16b, \x2\().16b, v18.16b
+ eor \x3\().16b, \x3\().16b, v19.16b
+ eor \x4\().16b, \x4\().16b, v20.16b
+ eor \x5\().16b, \x5\().16b, v21.16b
+ eor \x6\().16b, \x6\().16b, v22.16b
+ eor \x7\().16b, \x7\().16b, v23.16b
+ .endm
+
+ .macro shift_rows, x0, x1, x2, x3, x4, x5, x6, x7, mask
+ tbl \x0\().16b, {\x0\().16b}, \mask\().16b
+ tbl \x1\().16b, {\x1\().16b}, \mask\().16b
+ tbl \x2\().16b, {\x2\().16b}, \mask\().16b
+ tbl \x3\().16b, {\x3\().16b}, \mask\().16b
+ tbl \x4\().16b, {\x4\().16b}, \mask\().16b
+ tbl \x5\().16b, {\x5\().16b}, \mask\().16b
+ tbl \x6\().16b, {\x6\().16b}, \mask\().16b
+ tbl \x7\().16b, {\x7\().16b}, \mask\().16b
+ .endm
+
+ .macro mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
+ t0, t1, t2, t3, t4, t5, t6, t7, inv
+ ext \t0\().16b, \x0\().16b, \x0\().16b, #12
+ ext \t1\().16b, \x1\().16b, \x1\().16b, #12
+ eor \x0\().16b, \x0\().16b, \t0\().16b
+ ext \t2\().16b, \x2\().16b, \x2\().16b, #12
+ eor \x1\().16b, \x1\().16b, \t1\().16b
+ ext \t3\().16b, \x3\().16b, \x3\().16b, #12
+ eor \x2\().16b, \x2\().16b, \t2\().16b
+ ext \t4\().16b, \x4\().16b, \x4\().16b, #12
+ eor \x3\().16b, \x3\().16b, \t3\().16b
+ ext \t5\().16b, \x5\().16b, \x5\().16b, #12
+ eor \x4\().16b, \x4\().16b, \t4\().16b
+ ext \t6\().16b, \x6\().16b, \x6\().16b, #12
+ eor \x5\().16b, \x5\().16b, \t5\().16b
+ ext \t7\().16b, \x7\().16b, \x7\().16b, #12
+ eor \x6\().16b, \x6\().16b, \t6\().16b
+ eor \t1\().16b, \t1\().16b, \x0\().16b
+ eor \x7\().16b, \x7\().16b, \t7\().16b
+ ext \x0\().16b, \x0\().16b, \x0\().16b, #8
+ eor \t2\().16b, \t2\().16b, \x1\().16b
+ eor \t0\().16b, \t0\().16b, \x7\().16b
+ eor \t1\().16b, \t1\().16b, \x7\().16b
+ ext \x1\().16b, \x1\().16b, \x1\().16b, #8
+ eor \t5\().16b, \t5\().16b, \x4\().16b
+ eor \x0\().16b, \x0\().16b, \t0\().16b
+ eor \t6\().16b, \t6\().16b, \x5\().16b
+ eor \x1\().16b, \x1\().16b, \t1\().16b
+ ext \t0\().16b, \x4\().16b, \x4\().16b, #8
+ eor \t4\().16b, \t4\().16b, \x3\().16b
+ ext \t1\().16b, \x5\().16b, \x5\().16b, #8
+ eor \t7\().16b, \t7\().16b, \x6\().16b
+ ext \x4\().16b, \x3\().16b, \x3\().16b, #8
+ eor \t3\().16b, \t3\().16b, \x2\().16b
+ ext \x5\().16b, \x7\().16b, \x7\().16b, #8
+ eor \t4\().16b, \t4\().16b, \x7\().16b
+ ext \x3\().16b, \x6\().16b, \x6\().16b, #8
+ eor \t3\().16b, \t3\().16b, \x7\().16b
+ ext \x6\().16b, \x2\().16b, \x2\().16b, #8
+ eor \x7\().16b, \t1\().16b, \t5\().16b
+ .ifb \inv
+ eor \x2\().16b, \t0\().16b, \t4\().16b
+ eor \x4\().16b, \x4\().16b, \t3\().16b
+ eor \x5\().16b, \x5\().16b, \t7\().16b
+ eor \x3\().16b, \x3\().16b, \t6\().16b
+ eor \x6\().16b, \x6\().16b, \t2\().16b
+ .else
+ eor \t3\().16b, \t3\().16b, \x4\().16b
+ eor \x5\().16b, \x5\().16b, \t7\().16b
+ eor \x2\().16b, \x3\().16b, \t6\().16b
+ eor \x3\().16b, \t0\().16b, \t4\().16b
+ eor \x4\().16b, \x6\().16b, \t2\().16b
+ mov \x6\().16b, \t3\().16b
+ .endif
+ .endm
+
+ .macro inv_mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
+ t0, t1, t2, t3, t4, t5, t6, t7
+ ext \t0\().16b, \x0\().16b, \x0\().16b, #8
+ ext \t6\().16b, \x6\().16b, \x6\().16b, #8
+ ext \t7\().16b, \x7\().16b, \x7\().16b, #8
+ eor \t0\().16b, \t0\().16b, \x0\().16b
+ ext \t1\().16b, \x1\().16b, \x1\().16b, #8
+ eor \t6\().16b, \t6\().16b, \x6\().16b
+ ext \t2\().16b, \x2\().16b, \x2\().16b, #8
+ eor \t7\().16b, \t7\().16b, \x7\().16b
+ ext \t3\().16b, \x3\().16b, \x3\().16b, #8
+ eor \t1\().16b, \t1\().16b, \x1\().16b
+ ext \t4\().16b, \x4\().16b, \x4\().16b, #8
+ eor \t2\().16b, \t2\().16b, \x2\().16b
+ ext \t5\().16b, \x5\().16b, \x5\().16b, #8
+ eor \t3\().16b, \t3\().16b, \x3\().16b
+ eor \t4\().16b, \t4\().16b, \x4\().16b
+ eor \t5\().16b, \t5\().16b, \x5\().16b
+ eor \x0\().16b, \x0\().16b, \t6\().16b
+ eor \x1\().16b, \x1\().16b, \t6\().16b
+ eor \x2\().16b, \x2\().16b, \t0\().16b
+ eor \x4\().16b, \x4\().16b, \t2\().16b
+ eor \x3\().16b, \x3\().16b, \t1\().16b
+ eor \x1\().16b, \x1\().16b, \t7\().16b
+ eor \x2\().16b, \x2\().16b, \t7\().16b
+ eor \x4\().16b, \x4\().16b, \t6\().16b
+ eor \x5\().16b, \x5\().16b, \t3\().16b
+ eor \x3\().16b, \x3\().16b, \t6\().16b
+ eor \x6\().16b, \x6\().16b, \t4\().16b
+ eor \x4\().16b, \x4\().16b, \t7\().16b
+ eor \x5\().16b, \x5\().16b, \t7\().16b
+ eor \x7\().16b, \x7\().16b, \t5\().16b
+ mix_cols \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
+ \t0, \t1, \t2, \t3, \t4, \t5, \t6, \t7, 1
+ .endm
+
+ .macro swapmove_2x, a0, b0, a1, b1, n, mask, t0, t1
+ ushr \t0\().2d, \b0\().2d, #\n
+ ushr \t1\().2d, \b1\().2d, #\n
+ eor \t0\().16b, \t0\().16b, \a0\().16b
+ eor \t1\().16b, \t1\().16b, \a1\().16b
+ and \t0\().16b, \t0\().16b, \mask\().16b
+ and \t1\().16b, \t1\().16b, \mask\().16b
+ eor \a0\().16b, \a0\().16b, \t0\().16b
+ shl \t0\().2d, \t0\().2d, #\n
+ eor \a1\().16b, \a1\().16b, \t1\().16b
+ shl \t1\().2d, \t1\().2d, #\n
+ eor \b0\().16b, \b0\().16b, \t0\().16b
+ eor \b1\().16b, \b1\().16b, \t1\().16b
+ .endm
+
+ .macro bitslice, x7, x6, x5, x4, x3, x2, x1, x0, t0, t1, t2, t3
+ movi \t0\().16b, #0x55
+ movi \t1\().16b, #0x33
+ swapmove_2x \x0, \x1, \x2, \x3, 1, \t0, \t2, \t3
+ swapmove_2x \x4, \x5, \x6, \x7, 1, \t0, \t2, \t3
+ movi \t0\().16b, #0x0f
+ swapmove_2x \x0, \x2, \x1, \x3, 2, \t1, \t2, \t3
+ swapmove_2x \x4, \x6, \x5, \x7, 2, \t1, \t2, \t3
+ swapmove_2x \x0, \x4, \x1, \x5, 4, \t0, \t2, \t3
+ swapmove_2x \x2, \x6, \x3, \x7, 4, \t0, \t2, \t3
+ .endm
+
+
+ .align 6
+M0: .octa 0x0004080c0105090d02060a0e03070b0f
+
+M0SR: .octa 0x0004080c05090d010a0e02060f03070b
+SR: .octa 0x0f0e0d0c0a09080b0504070600030201
+SRM0: .octa 0x01060b0c0207080d0304090e00050a0f
+
+M0ISR: .octa 0x0004080c0d0105090a0e0206070b0f03
+ISR: .octa 0x0f0e0d0c080b0a090504070602010003
+ISRM0: .octa 0x0306090c00070a0d01040b0e0205080f
+
+ /*
+ * void aesbs_convert_key(u8 out[], u32 const rk[], int rounds)
+ */
+ENTRY(aesbs_convert_key)
+ ld1 {v7.4s}, [x1], #16 // load round 0 key
+ ld1 {v17.4s}, [x1], #16 // load round 1 key
+
+ movi v8.16b, #0x01 // bit masks
+ movi v9.16b, #0x02
+ movi v10.16b, #0x04
+ movi v11.16b, #0x08
+ movi v12.16b, #0x10
+ movi v13.16b, #0x20
+ movi v14.16b, #0x40
+ movi v15.16b, #0x80
+ ldr q16, M0
+
+ sub x2, x2, #1
+ str q7, [x0], #16 // save round 0 key
+
+.Lkey_loop:
+ tbl v7.16b ,{v17.16b}, v16.16b
+ ld1 {v17.4s}, [x1], #16 // load next round key
+
+ cmtst v0.16b, v7.16b, v8.16b
+ cmtst v1.16b, v7.16b, v9.16b
+ cmtst v2.16b, v7.16b, v10.16b
+ cmtst v3.16b, v7.16b, v11.16b
+ cmtst v4.16b, v7.16b, v12.16b
+ cmtst v5.16b, v7.16b, v13.16b
+ cmtst v6.16b, v7.16b, v14.16b
+ cmtst v7.16b, v7.16b, v15.16b
+ not v0.16b, v0.16b
+ not v1.16b, v1.16b
+ not v5.16b, v5.16b
+ not v6.16b, v6.16b
+
+ subs x2, x2, #1
+ stp q0, q1, [x0], #128
+ stp q2, q3, [x0, #-96]
+ stp q4, q5, [x0, #-64]
+ stp q6, q7, [x0, #-32]
+ b.ne .Lkey_loop
+
+ movi v7.16b, #0x63 // compose .L63
+ eor v17.16b, v17.16b, v7.16b
+ str q17, [x0]
+ ret
+ENDPROC(aesbs_convert_key)
+
+ .align 4
+aesbs_encrypt8:
+ ldr q9, [bskey], #16 // round 0 key
+ ldr q8, M0SR
+ ldr q24, SR
+
+ eor v10.16b, v0.16b, v9.16b // xor with round0 key
+ eor v11.16b, v1.16b, v9.16b
+ tbl v0.16b, {v10.16b}, v8.16b
+ eor v12.16b, v2.16b, v9.16b
+ tbl v1.16b, {v11.16b}, v8.16b
+ eor v13.16b, v3.16b, v9.16b
+ tbl v2.16b, {v12.16b}, v8.16b
+ eor v14.16b, v4.16b, v9.16b
+ tbl v3.16b, {v13.16b}, v8.16b
+ eor v15.16b, v5.16b, v9.16b
+ tbl v4.16b, {v14.16b}, v8.16b
+ eor v10.16b, v6.16b, v9.16b
+ tbl v5.16b, {v15.16b}, v8.16b
+ eor v11.16b, v7.16b, v9.16b
+ tbl v6.16b, {v10.16b}, v8.16b
+ tbl v7.16b, {v11.16b}, v8.16b
+
+ bitslice v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11
+
+ sub rounds, rounds, #1
+ b .Lenc_sbox
+
+.Lenc_loop:
+ shift_rows v0, v1, v2, v3, v4, v5, v6, v7, v24
+.Lenc_sbox:
+ sbox v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, \
+ v13, v14, v15
+ subs rounds, rounds, #1
+ b.cc .Lenc_done
+
+ enc_next_rk
+
+ mix_cols v0, v1, v4, v6, v3, v7, v2, v5, v8, v9, v10, v11, v12, \
+ v13, v14, v15
+
+ add_round_key v0, v1, v2, v3, v4, v5, v6, v7
+
+ b.ne .Lenc_loop
+ ldr q24, SRM0
+ b .Lenc_loop
+
+.Lenc_done:
+ ldr q12, [bskey] // last round key
+
+ bitslice v0, v1, v4, v6, v3, v7, v2, v5, v8, v9, v10, v11
+
+ eor v0.16b, v0.16b, v12.16b
+ eor v1.16b, v1.16b, v12.16b
+ eor v4.16b, v4.16b, v12.16b
+ eor v6.16b, v6.16b, v12.16b
+ eor v3.16b, v3.16b, v12.16b
+ eor v7.16b, v7.16b, v12.16b
+ eor v2.16b, v2.16b, v12.16b
+ eor v5.16b, v5.16b, v12.16b
+ ret
+ENDPROC(aesbs_encrypt8)
+
+ .align 4
+aesbs_decrypt8:
+ lsl x9, rounds, #7
+ add bskey, bskey, x9
+
+ ldr q9, [bskey, #-112]! // round 0 key
+ ldr q8, M0ISR
+ ldr q24, ISR
+
+ eor v10.16b, v0.16b, v9.16b // xor with round0 key
+ eor v11.16b, v1.16b, v9.16b
+ tbl v0.16b, {v10.16b}, v8.16b
+ eor v12.16b, v2.16b, v9.16b
+ tbl v1.16b, {v11.16b}, v8.16b
+ eor v13.16b, v3.16b, v9.16b
+ tbl v2.16b, {v12.16b}, v8.16b
+ eor v14.16b, v4.16b, v9.16b
+ tbl v3.16b, {v13.16b}, v8.16b
+ eor v15.16b, v5.16b, v9.16b
+ tbl v4.16b, {v14.16b}, v8.16b
+ eor v10.16b, v6.16b, v9.16b
+ tbl v5.16b, {v15.16b}, v8.16b
+ eor v11.16b, v7.16b, v9.16b
+ tbl v6.16b, {v10.16b}, v8.16b
+ tbl v7.16b, {v11.16b}, v8.16b
+
+ bitslice v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11
+
+ sub rounds, rounds, #1
+ b .Ldec_sbox
+
+.Ldec_loop:
+ shift_rows v0, v1, v2, v3, v4, v5, v6, v7, v24
+.Ldec_sbox:
+ inv_sbox v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, \
+ v13, v14, v15
+ subs rounds, rounds, #1
+ b.cc .Ldec_done
+
+ dec_next_rk
+
+ add_round_key v0, v1, v6, v4, v2, v7, v3, v5
+
+ inv_mix_cols v0, v1, v6, v4, v2, v7, v3, v5, v8, v9, v10, v11, v12, \
+ v13, v14, v15
+
+ b.ne .Ldec_loop
+ ldr q24, ISRM0
+ b .Ldec_loop
+.Ldec_done:
+ ldr q12, [bskey, #-16] // last round key
+
+ bitslice v0, v1, v6, v4, v2, v7, v3, v5, v8, v9, v10, v11
+
+ eor v0.16b, v0.16b, v12.16b
+ eor v1.16b, v1.16b, v12.16b
+ eor v6.16b, v6.16b, v12.16b
+ eor v4.16b, v4.16b, v12.16b
+ eor v2.16b, v2.16b, v12.16b
+ eor v7.16b, v7.16b, v12.16b
+ eor v3.16b, v3.16b, v12.16b
+ eor v5.16b, v5.16b, v12.16b
+ ret
+ENDPROC(aesbs_decrypt8)
+
+ /*
+ * aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+ * int blocks)
+ * aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+ * int blocks)
+ */
+ .macro __ecb_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
+ stp x29, x30, [sp, #-16]!
+ mov x29, sp
+
+99: mov x5, #1
+ lsl x5, x5, x4
+ subs w4, w4, #8
+ csel x4, x4, xzr, pl
+ csel x5, x5, xzr, mi
+
+ ld1 {v0.16b}, [x1], #16
+ tbnz x5, #1, 0f
+ ld1 {v1.16b}, [x1], #16
+ tbnz x5, #2, 0f
+ ld1 {v2.16b}, [x1], #16
+ tbnz x5, #3, 0f
+ ld1 {v3.16b}, [x1], #16
+ tbnz x5, #4, 0f
+ ld1 {v4.16b}, [x1], #16
+ tbnz x5, #5, 0f
+ ld1 {v5.16b}, [x1], #16
+ tbnz x5, #6, 0f
+ ld1 {v6.16b}, [x1], #16
+ tbnz x5, #7, 0f
+ ld1 {v7.16b}, [x1], #16
+
+0: mov bskey, x2
+ mov rounds, x3
+ bl \do8
+
+ st1 {\o0\().16b}, [x0], #16
+ tbnz x5, #1, 1f
+ st1 {\o1\().16b}, [x0], #16
+ tbnz x5, #2, 1f
+ st1 {\o2\().16b}, [x0], #16
+ tbnz x5, #3, 1f
+ st1 {\o3\().16b}, [x0], #16
+ tbnz x5, #4, 1f
+ st1 {\o4\().16b}, [x0], #16
+ tbnz x5, #5, 1f
+ st1 {\o5\().16b}, [x0], #16
+ tbnz x5, #6, 1f
+ st1 {\o6\().16b}, [x0], #16
+ tbnz x5, #7, 1f
+ st1 {\o7\().16b}, [x0], #16
+
+ cbnz x4, 99b
+
+1: ldp x29, x30, [sp], #16
+ ret
+ .endm
+
+ .align 4
+ENTRY(aesbs_ecb_encrypt)
+ __ecb_crypt aesbs_encrypt8, v0, v1, v4, v6, v3, v7, v2, v5
+ENDPROC(aesbs_ecb_encrypt)
+
+ .align 4
+ENTRY(aesbs_ecb_decrypt)
+ __ecb_crypt aesbs_decrypt8, v0, v1, v6, v4, v2, v7, v3, v5
+ENDPROC(aesbs_ecb_decrypt)
+
+ /*
+ * aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+ * int blocks, u8 iv[])
+ */
+ .align 4
+ENTRY(aesbs_cbc_decrypt)
+ stp x29, x30, [sp, #-16]!
+ mov x29, sp
+
+99: mov x6, #1
+ lsl x6, x6, x4
+ subs w4, w4, #8
+ csel x4, x4, xzr, pl
+ csel x6, x6, xzr, mi
+
+ ld1 {v0.16b}, [x1], #16
+ mov v25.16b, v0.16b
+ tbnz x6, #1, 0f
+ ld1 {v1.16b}, [x1], #16
+ mov v26.16b, v1.16b
+ tbnz x6, #2, 0f
+ ld1 {v2.16b}, [x1], #16
+ mov v27.16b, v2.16b
+ tbnz x6, #3, 0f
+ ld1 {v3.16b}, [x1], #16
+ mov v28.16b, v3.16b
+ tbnz x6, #4, 0f
+ ld1 {v4.16b}, [x1], #16
+ mov v29.16b, v4.16b
+ tbnz x6, #5, 0f
+ ld1 {v5.16b}, [x1], #16
+ mov v30.16b, v5.16b
+ tbnz x6, #6, 0f
+ ld1 {v6.16b}, [x1], #16
+ mov v31.16b, v6.16b
+ tbnz x6, #7, 0f
+ ld1 {v7.16b}, [x1]
+
+0: mov bskey, x2
+ mov rounds, x3
+ bl aesbs_decrypt8
+
+ ld1 {v24.16b}, [x5] // load IV
+
+ eor v1.16b, v1.16b, v25.16b
+ eor v6.16b, v6.16b, v26.16b
+ eor v4.16b, v4.16b, v27.16b
+ eor v2.16b, v2.16b, v28.16b
+ eor v7.16b, v7.16b, v29.16b
+ eor v0.16b, v0.16b, v24.16b
+ eor v3.16b, v3.16b, v30.16b
+ eor v5.16b, v5.16b, v31.16b
+
+ st1 {v0.16b}, [x0], #16
+ mov v24.16b, v25.16b
+ tbnz x6, #1, 1f
+ st1 {v1.16b}, [x0], #16
+ mov v24.16b, v26.16b
+ tbnz x6, #2, 1f
+ st1 {v6.16b}, [x0], #16
+ mov v24.16b, v27.16b
+ tbnz x6, #3, 1f
+ st1 {v4.16b}, [x0], #16
+ mov v24.16b, v28.16b
+ tbnz x6, #4, 1f
+ st1 {v2.16b}, [x0], #16
+ mov v24.16b, v29.16b
+ tbnz x6, #5, 1f
+ st1 {v7.16b}, [x0], #16
+ mov v24.16b, v30.16b
+ tbnz x6, #6, 1f
+ st1 {v3.16b}, [x0], #16
+ mov v24.16b, v31.16b
+ tbnz x6, #7, 1f
+ ld1 {v24.16b}, [x1], #16
+ st1 {v5.16b}, [x0], #16
+1: st1 {v24.16b}, [x5] // store IV
+
+ cbnz x4, 99b
+
+ ldp x29, x30, [sp], #16
+ ret
+ENDPROC(aesbs_cbc_decrypt)
+
+ .macro next_tweak, out, in, const, tmp
+ sshr \tmp\().2d, \in\().2d, #63
+ and \tmp\().16b, \tmp\().16b, \const\().16b
+ add \out\().2d, \in\().2d, \in\().2d
+ ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8
+ eor \out\().16b, \out\().16b, \tmp\().16b
+ .endm
+
+ .align 4
+.Lxts_mul_x:
+CPU_LE( .quad 1, 0x87 )
+CPU_BE( .quad 0x87, 1 )
+
+ /*
+ * aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+ * int blocks, u8 iv[])
+ * aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+ * int blocks, u8 iv[])
+ */
+__xts_crypt8:
+ mov x6, #1
+ lsl x6, x6, x4
+ subs w4, w4, #8
+ csel x4, x4, xzr, pl
+ csel x6, x6, xzr, mi
+
+ ld1 {v0.16b}, [x1], #16
+ next_tweak v26, v25, v30, v31
+ eor v0.16b, v0.16b, v25.16b
+ tbnz x6, #1, 0f
+
+ ld1 {v1.16b}, [x1], #16
+ next_tweak v27, v26, v30, v31
+ eor v1.16b, v1.16b, v26.16b
+ tbnz x6, #2, 0f
+
+ ld1 {v2.16b}, [x1], #16
+ next_tweak v28, v27, v30, v31
+ eor v2.16b, v2.16b, v27.16b
+ tbnz x6, #3, 0f
+
+ ld1 {v3.16b}, [x1], #16
+ next_tweak v29, v28, v30, v31
+ eor v3.16b, v3.16b, v28.16b
+ tbnz x6, #4, 0f
+
+ ld1 {v4.16b}, [x1], #16
+ str q29, [sp, #16]
+ eor v4.16b, v4.16b, v29.16b
+ next_tweak v29, v29, v30, v31
+ tbnz x6, #5, 0f
+
+ ld1 {v5.16b}, [x1], #16
+ str q29, [sp, #32]
+ eor v5.16b, v5.16b, v29.16b
+ next_tweak v29, v29, v30, v31
+ tbnz x6, #6, 0f
+
+ ld1 {v6.16b}, [x1], #16
+ str q29, [sp, #48]
+ eor v6.16b, v6.16b, v29.16b
+ next_tweak v29, v29, v30, v31
+ tbnz x6, #7, 0f
+
+ ld1 {v7.16b}, [x1], #16
+ str q29, [sp, #64]
+ eor v7.16b, v7.16b, v29.16b
+ next_tweak v29, v29, v30, v31
+
+0: mov bskey, x2
+ mov rounds, x3
+ br x7
+ENDPROC(__xts_crypt8)
+
+ .macro __xts_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
+ stp x29, x30, [sp, #-80]!
+ mov x29, sp
+
+ ldr q30, .Lxts_mul_x
+ ld1 {v25.16b}, [x5]
+
+99: adr x7, \do8
+ bl __xts_crypt8
+
+ ldp q16, q17, [sp, #16]
+ ldp q18, q19, [sp, #48]
+
+ eor \o0\().16b, \o0\().16b, v25.16b
+ eor \o1\().16b, \o1\().16b, v26.16b
+ eor \o2\().16b, \o2\().16b, v27.16b
+ eor \o3\().16b, \o3\().16b, v28.16b
+
+ st1 {\o0\().16b}, [x0], #16
+ mov v25.16b, v26.16b
+ tbnz x6, #1, 1f
+ st1 {\o1\().16b}, [x0], #16
+ mov v25.16b, v27.16b
+ tbnz x6, #2, 1f
+ st1 {\o2\().16b}, [x0], #16
+ mov v25.16b, v28.16b
+ tbnz x6, #3, 1f
+ st1 {\o3\().16b}, [x0], #16
+ mov v25.16b, v29.16b
+ tbnz x6, #4, 1f
+
+ eor \o4\().16b, \o4\().16b, v16.16b
+ eor \o5\().16b, \o5\().16b, v17.16b
+ eor \o6\().16b, \o6\().16b, v18.16b
+ eor \o7\().16b, \o7\().16b, v19.16b
+
+ st1 {\o4\().16b}, [x0], #16
+ tbnz x6, #5, 1f
+ st1 {\o5\().16b}, [x0], #16
+ tbnz x6, #6, 1f
+ st1 {\o6\().16b}, [x0], #16
+ tbnz x6, #7, 1f
+ st1 {\o7\().16b}, [x0], #16
+
+ cbnz x4, 99b
+
+1: st1 {v25.16b}, [x5]
+ ldp x29, x30, [sp], #80
+ ret
+ .endm
+
+ENTRY(aesbs_xts_encrypt)
+ __xts_crypt aesbs_encrypt8, v0, v1, v4, v6, v3, v7, v2, v5
+ENDPROC(aesbs_xts_encrypt)
+
+ENTRY(aesbs_xts_decrypt)
+ __xts_crypt aesbs_decrypt8, v0, v1, v6, v4, v2, v7, v3, v5
+ENDPROC(aesbs_xts_decrypt)
+
+ .macro next_ctr, v
+ mov \v\().d[1], x8
+ adds x8, x8, #1
+ mov \v\().d[0], x7
+ adc x7, x7, xzr
+ rev64 \v\().16b, \v\().16b
+ .endm
+
+ /*
+ * aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
+ * int rounds, int blocks, u8 iv[], bool final)
+ */
+ENTRY(aesbs_ctr_encrypt)
+ stp x29, x30, [sp, #-16]!
+ mov x29, sp
+
+ add x4, x4, x6 // do one extra block if final
+
+ ldp x7, x8, [x5]
+ ld1 {v0.16b}, [x5]
+CPU_LE( rev x7, x7 )
+CPU_LE( rev x8, x8 )
+ adds x8, x8, #1
+ adc x7, x7, xzr
+
+99: mov x9, #1
+ lsl x9, x9, x4
+ subs w4, w4, #8
+ csel x4, x4, xzr, pl
+ csel x9, x9, xzr, le
+
+ next_ctr v1
+ next_ctr v2
+ next_ctr v3
+ next_ctr v4
+ next_ctr v5
+ next_ctr v6
+ next_ctr v7
+
+0: mov bskey, x2
+ mov rounds, x3
+ bl aesbs_encrypt8
+
+ lsr x9, x9, x6 // disregard the extra block
+ tbnz x9, #0, 0f
+
+ ld1 {v8.16b}, [x1], #16
+ eor v0.16b, v0.16b, v8.16b
+ st1 {v0.16b}, [x0], #16
+ tbnz x9, #1, 1f
+
+ ld1 {v9.16b}, [x1], #16
+ eor v1.16b, v1.16b, v9.16b
+ st1 {v1.16b}, [x0], #16
+ tbnz x9, #2, 2f
+
+ ld1 {v10.16b}, [x1], #16
+ eor v4.16b, v4.16b, v10.16b
+ st1 {v4.16b}, [x0], #16
+ tbnz x9, #3, 3f
+
+ ld1 {v11.16b}, [x1], #16
+ eor v6.16b, v6.16b, v11.16b
+ st1 {v6.16b}, [x0], #16
+ tbnz x9, #4, 4f
+
+ ld1 {v12.16b}, [x1], #16
+ eor v3.16b, v3.16b, v12.16b
+ st1 {v3.16b}, [x0], #16
+ tbnz x9, #5, 5f
+
+ ld1 {v13.16b}, [x1], #16
+ eor v7.16b, v7.16b, v13.16b
+ st1 {v7.16b}, [x0], #16
+ tbnz x9, #6, 6f
+
+ ld1 {v14.16b}, [x1], #16
+ eor v2.16b, v2.16b, v14.16b
+ st1 {v2.16b}, [x0], #16
+ tbnz x9, #7, 7f
+
+ ld1 {v15.16b}, [x1], #16
+ eor v5.16b, v5.16b, v15.16b
+ st1 {v5.16b}, [x0], #16
+
+ next_ctr v0
+ cbnz x4, 99b
+
+0: st1 {v0.16b}, [x5]
+8: ldp x29, x30, [sp], #16
+ ret
+
+ /*
+ * If we are handling the tail of the input (x6 == 1), return the
+ * final keystream block back to the caller via the IV buffer.
+ */
+1: cbz x6, 8b
+ st1 {v1.16b}, [x5]
+ b 8b
+2: cbz x6, 8b
+ st1 {v4.16b}, [x5]
+ b 8b
+3: cbz x6, 8b
+ st1 {v6.16b}, [x5]
+ b 8b
+4: cbz x6, 8b
+ st1 {v3.16b}, [x5]
+ b 8b
+5: cbz x6, 8b
+ st1 {v7.16b}, [x5]
+ b 8b
+6: cbz x6, 8b
+ st1 {v2.16b}, [x5]
+ b 8b
+7: cbz x6, 8b
+ st1 {v5.16b}, [x5]
+ b 8b
+ENDPROC(aesbs_ctr_encrypt)
diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c
new file mode 100644
index 000000000000..323dd76ae5f0
--- /dev/null
+++ b/arch/arm64/crypto/aes-neonbs-glue.c
@@ -0,0 +1,420 @@
+/*
+ * Bit sliced AES using NEON instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/neon.h>
+#include <crypto/aes.h>
+#include <crypto/cbc.h>
+#include <crypto/internal/simd.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/xts.h>
+#include <linux/module.h>
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+
+MODULE_ALIAS_CRYPTO("ecb(aes)");
+MODULE_ALIAS_CRYPTO("cbc(aes)");
+MODULE_ALIAS_CRYPTO("ctr(aes)");
+MODULE_ALIAS_CRYPTO("xts(aes)");
+
+asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds);
+
+asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks);
+asmlinkage void aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks);
+
+asmlinkage void aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks, u8 iv[]);
+
+asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks, u8 iv[], bool final);
+
+asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks, u8 iv[]);
+asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks, u8 iv[]);
+
+asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
+
+struct aesbs_ctx {
+ u8 rk[13 * (8 * AES_BLOCK_SIZE) + 32];
+ int rounds;
+} __aligned(AES_BLOCK_SIZE);
+
+struct aesbs_cbc_ctx {
+ struct aesbs_ctx key;
+ u32 enc[AES_MAX_KEYLENGTH_U32];
+};
+
+struct aesbs_xts_ctx {
+ struct aesbs_ctx key;
+ u32 twkey[AES_MAX_KEYLENGTH_U32];
+};
+
+static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct crypto_aes_ctx rk;
+ int err;
+
+ err = crypto_aes_expand_key(&rk, in_key, key_len);
+ if (err)
+ return err;
+
+ ctx->rounds = 6 + key_len / 4;
+
+ kernel_neon_begin();
+ aesbs_convert_key(ctx->rk, rk.key_enc, ctx->rounds);
+ kernel_neon_end();
+
+ return 0;
+}
+
+static int __ecb_crypt(struct skcipher_request *req,
+ void (*fn)(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks))
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ kernel_neon_begin();
+ while (walk.nbytes >= AES_BLOCK_SIZE) {
+ unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
+
+ if (walk.nbytes < walk.total)
+ blocks = round_down(blocks,
+ walk.stride / AES_BLOCK_SIZE);
+
+ fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk,
+ ctx->rounds, blocks);
+ err = skcipher_walk_done(&walk,
+ walk.nbytes - blocks * AES_BLOCK_SIZE);
+ }
+ kernel_neon_end();
+
+ return err;
+}
+
+static int ecb_encrypt(struct skcipher_request *req)
+{
+ return __ecb_crypt(req, aesbs_ecb_encrypt);
+}
+
+static int ecb_decrypt(struct skcipher_request *req)
+{
+ return __ecb_crypt(req, aesbs_ecb_decrypt);
+}
+
+static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct crypto_aes_ctx rk;
+ int err;
+
+ err = crypto_aes_expand_key(&rk, in_key, key_len);
+ if (err)
+ return err;
+
+ ctx->key.rounds = 6 + key_len / 4;
+
+ memcpy(ctx->enc, rk.key_enc, sizeof(ctx->enc));
+
+ kernel_neon_begin();
+ aesbs_convert_key(ctx->key.rk, rk.key_enc, ctx->key.rounds);
+ kernel_neon_end();
+
+ return 0;
+}
+
+static void cbc_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst)
+{
+ struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ __aes_arm64_encrypt(ctx->enc, dst, src, ctx->key.rounds);
+}
+
+static int cbc_encrypt(struct skcipher_request *req)
+{
+ return crypto_cbc_encrypt_walk(req, cbc_encrypt_one);
+}
+
+static int cbc_decrypt(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ kernel_neon_begin();
+ while (walk.nbytes >= AES_BLOCK_SIZE) {
+ unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
+
+ if (walk.nbytes < walk.total)
+ blocks = round_down(blocks,
+ walk.stride / AES_BLOCK_SIZE);
+
+ aesbs_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
+ ctx->key.rk, ctx->key.rounds, blocks,
+ walk.iv);
+ err = skcipher_walk_done(&walk,
+ walk.nbytes - blocks * AES_BLOCK_SIZE);
+ }
+ kernel_neon_end();
+
+ return err;
+}
+
+static int ctr_encrypt(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ kernel_neon_begin();
+ while (walk.nbytes > 0) {
+ unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
+ bool final = (walk.total % AES_BLOCK_SIZE) != 0;
+
+ if (walk.nbytes < walk.total) {
+ blocks = round_down(blocks,
+ walk.stride / AES_BLOCK_SIZE);
+ final = false;
+ }
+
+ aesbs_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
+ ctx->rk, ctx->rounds, blocks, walk.iv, final);
+
+ if (final) {
+ u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
+ u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
+
+ if (dst != src)
+ memcpy(dst, src, walk.total % AES_BLOCK_SIZE);
+ crypto_xor(dst, walk.iv, walk.total % AES_BLOCK_SIZE);
+
+ err = skcipher_walk_done(&walk, 0);
+ break;
+ }
+ err = skcipher_walk_done(&walk,
+ walk.nbytes - blocks * AES_BLOCK_SIZE);
+ }
+ kernel_neon_end();
+
+ return err;
+}
+
+static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct crypto_aes_ctx rk;
+ int err;
+
+ err = xts_verify_key(tfm, in_key, key_len);
+ if (err)
+ return err;
+
+ key_len /= 2;
+ err = crypto_aes_expand_key(&rk, in_key + key_len, key_len);
+ if (err)
+ return err;
+
+ memcpy(ctx->twkey, rk.key_enc, sizeof(ctx->twkey));
+
+ return aesbs_setkey(tfm, in_key, key_len);
+}
+
+static int __xts_crypt(struct skcipher_request *req,
+ void (*fn)(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks, u8 iv[]))
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ __aes_arm64_encrypt(ctx->twkey, walk.iv, walk.iv, ctx->key.rounds);
+
+ kernel_neon_begin();
+ while (walk.nbytes >= AES_BLOCK_SIZE) {
+ unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
+
+ if (walk.nbytes < walk.total)
+ blocks = round_down(blocks,
+ walk.stride / AES_BLOCK_SIZE);
+
+ fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->key.rk,
+ ctx->key.rounds, blocks, walk.iv);
+ err = skcipher_walk_done(&walk,
+ walk.nbytes - blocks * AES_BLOCK_SIZE);
+ }
+ kernel_neon_end();
+
+ return err;
+}
+
+static int xts_encrypt(struct skcipher_request *req)
+{
+ return __xts_crypt(req, aesbs_xts_encrypt);
+}
+
+static int xts_decrypt(struct skcipher_request *req)
+{
+ return __xts_crypt(req, aesbs_xts_decrypt);
+}
+
+static struct skcipher_alg aes_algs[] = { {
+ .base.cra_name = "__ecb(aes)",
+ .base.cra_driver_name = "__ecb-aes-neonbs",
+ .base.cra_priority = 250,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct aesbs_ctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_flags = CRYPTO_ALG_INTERNAL,
+
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
+ .setkey = aesbs_setkey,
+ .encrypt = ecb_encrypt,
+ .decrypt = ecb_decrypt,
+}, {
+ .base.cra_name = "__cbc(aes)",
+ .base.cra_driver_name = "__cbc-aes-neonbs",
+ .base.cra_priority = 250,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_flags = CRYPTO_ALG_INTERNAL,
+
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aesbs_cbc_setkey,
+ .encrypt = cbc_encrypt,
+ .decrypt = cbc_decrypt,
+}, {
+ .base.cra_name = "__ctr(aes)",
+ .base.cra_driver_name = "__ctr-aes-neonbs",
+ .base.cra_priority = 250,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct aesbs_ctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_flags = CRYPTO_ALG_INTERNAL,
+
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .chunksize = AES_BLOCK_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aesbs_setkey,
+ .encrypt = ctr_encrypt,
+ .decrypt = ctr_encrypt,
+}, {
+ .base.cra_name = "ctr(aes)",
+ .base.cra_driver_name = "ctr-aes-neonbs",
+ .base.cra_priority = 250 - 1,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct aesbs_ctx),
+ .base.cra_module = THIS_MODULE,
+
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .chunksize = AES_BLOCK_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aesbs_setkey,
+ .encrypt = ctr_encrypt,
+ .decrypt = ctr_encrypt,
+}, {
+ .base.cra_name = "__xts(aes)",
+ .base.cra_driver_name = "__xts-aes-neonbs",
+ .base.cra_priority = 250,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_flags = CRYPTO_ALG_INTERNAL,
+
+ .min_keysize = 2 * AES_MIN_KEY_SIZE,
+ .max_keysize = 2 * AES_MAX_KEY_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aesbs_xts_setkey,
+ .encrypt = xts_encrypt,
+ .decrypt = xts_decrypt,
+} };
+
+static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
+
+static void aes_exit(void)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
+ if (aes_simd_algs[i])
+ simd_skcipher_free(aes_simd_algs[i]);
+
+ crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
+}
+
+static int __init aes_init(void)
+{
+ struct simd_skcipher_alg *simd;
+ const char *basename;
+ const char *algname;
+ const char *drvname;
+ int err;
+ int i;
+
+ if (!(elf_hwcap & HWCAP_ASIMD))
+ return -ENODEV;
+
+ err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
+ if (err)
+ return err;
+
+ for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
+ if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
+ continue;
+
+ algname = aes_algs[i].base.cra_name + 2;
+ drvname = aes_algs[i].base.cra_driver_name + 2;
+ basename = aes_algs[i].base.cra_driver_name;
+ simd = simd_skcipher_create_compat(algname, drvname, basename);
+ err = PTR_ERR(simd);
+ if (IS_ERR(simd))
+ goto unregister_simds;
+
+ aes_simd_algs[i] = simd;
+ }
+ return 0;
+
+unregister_simds:
+ aes_exit();
+ return err;
+}
+
+module_init(aes_init);
+module_exit(aes_exit);
--
2.7.4
^ permalink raw reply related
* [PATCH] arm64: assembler: make adr_l work in modules under KASLR
From: Mark Rutland @ 2017-01-11 16:44 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <CAKv+Gu-FMCdkxpFCheCoRrCYFUdZPpViH4-G613sXyQVEH3chA@mail.gmail.com>
On Wed, Jan 11, 2017 at 04:08:30PM +0000, Ard Biesheuvel wrote:
> On 11 January 2017 at 15:34, Mark Rutland <mark.rutland@arm.com> wrote:
> > On Wed, Jan 11, 2017 at 03:25:09PM +0000, Ard Biesheuvel wrote:
> >> On 11 January 2017 at 15:18, Mark Rutland <mark.rutland@arm.com> wrote:
> >> > On Wed, Jan 11, 2017 at 02:54:53PM +0000, Ard Biesheuvel wrote:
> >> But in general, if the macro is available to modules, I would like to
> >> make sure that it does not result in code that builds fine but may
> >> fail in some cases only at runtime, especially given the fact that
> >> there is also a Cortex-A53 erratum regarding adrp instructions, for
> >> which reason we build modules with -mcmodel=large (which amounts to
> >> the same thing as the patch above)
> >>
> >> > It seems somewhat surprising to me to have adr_l expand to something
> >> > that doesn't use adr/adrp, but that's not necessarily a problem.
> >>
> >> I did realise that, but I don't think it is a problem tbh.
> >
> > In this case it should be fine, certainly.
> >
> > There are cases like the early boot code and hyp code where it's
> > critical that we use adr. It's also possible that we might build
> > (modular) drivers which want some idmapped code, where we want adr, so
> > it seems unfortunate that this depends on howthe code is built.
>
> How would /that/ work? Modules are vmalloc'ed, and not covered by the
> ID map to begin with, so it is impossible to execute those adr
> instructions in a way that would make them return anything other than
> the virtual address of the symbol they refer to.
That's a fair point.
> > So, maybe it's better to have a mov_sym helper for this case, to be
> > explicit about what we want? That can use either adr* or mov*, or the
> > latter consistently.
>
> Well, the point is that adr_l should not be used for modules so adding
> something that may be used is fine, but that still leaves the risk
> that someone may end up using it in a module.
Sure.
Given you have a concrete use case, and all I have are some vague
concerns for code that doesn't exist at present, I have no real
objection to the patch as it stands. FWIW:
Acked-by: Mark Rutland <mark.rutland@arm.com>
Thanks,
Mark.
^ permalink raw reply
* [PATCH] usb: dwc3 dwc3_exynos_probe() change goto labels to meaningful names
From: Shuah Khan @ 2017-01-11 16:45 UTC (permalink / raw)
To: linux-arm-kernel
Change goto labels to meaningful names from a series of errNs.
Signed-off-by: Shuah Khan <shuahkh@osg.samsung.com>
---
drivers/usb/dwc3/dwc3-exynos.c | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/drivers/usb/dwc3/dwc3-exynos.c b/drivers/usb/dwc3/dwc3-exynos.c
index f7421c2..ea50f74 100644
--- a/drivers/usb/dwc3/dwc3-exynos.c
+++ b/drivers/usb/dwc3/dwc3-exynos.c
@@ -147,53 +147,53 @@ static int dwc3_exynos_probe(struct platform_device *pdev)
exynos->vdd33 = devm_regulator_get(dev, "vdd33");
if (IS_ERR(exynos->vdd33)) {
ret = PTR_ERR(exynos->vdd33);
- goto err2;
+ goto vdd33_err;
}
ret = regulator_enable(exynos->vdd33);
if (ret) {
dev_err(dev, "Failed to enable VDD33 supply\n");
- goto err2;
+ goto vdd33_err;
}
exynos->vdd10 = devm_regulator_get(dev, "vdd10");
if (IS_ERR(exynos->vdd10)) {
ret = PTR_ERR(exynos->vdd10);
- goto err3;
+ goto vdd10_err;
}
ret = regulator_enable(exynos->vdd10);
if (ret) {
dev_err(dev, "Failed to enable VDD10 supply\n");
- goto err3;
+ goto vdd10_err;
}
ret = dwc3_exynos_register_phys(exynos);
if (ret) {
dev_err(dev, "couldn't register PHYs\n");
- goto err4;
+ goto phys_err;
}
if (node) {
ret = of_platform_populate(node, NULL, NULL, dev);
if (ret) {
dev_err(dev, "failed to add dwc3 core\n");
- goto err5;
+ goto populate_err;
}
} else {
dev_err(dev, "no device node, failed to add dwc3 core\n");
ret = -ENODEV;
- goto err5;
+ goto populate_err;
}
return 0;
-err5:
+populate_err:
platform_device_unregister(exynos->usb2_phy);
platform_device_unregister(exynos->usb3_phy);
-err4:
+phys_err:
regulator_disable(exynos->vdd10);
-err3:
+vdd10_err:
regulator_disable(exynos->vdd33);
-err2:
+vdd33_err:
clk_disable_unprepare(exynos->axius_clk);
axius_clk_err:
clk_disable_unprepare(exynos->susp_clk);
--
2.7.4
^ permalink raw reply related
* [PATCH] arm64: avoid increasing DMA masks above what hardware supports
From: Robin Murphy @ 2017-01-11 16:50 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <105b0b38-8c48-3355-4951-435c5dca18a4@cogentembedded.com>
On 11/01/17 16:03, Nikita Yushchenko wrote:
>
>
> 11.01.2017 17:50, Robin Murphy ?????:
>> On 11/01/17 13:41, Nikita Yushchenko wrote:
>>>> Yes, I think that ought to work, although the __iommu_setup_dma_ops()
>>>> call will still want a real size reflecting the default mask
>>>
>>> I see iommu_dma_ops do not define set_dma_mask.
>>>
>>> So what if setup was done for size reflecting one mask and then driver
>>> changes mask? Will things still operate correctly?
>>
>> We've overridden dma_set_mask() at the function level, so it should
>> always apply regardless. Besides, none of the arm64 ops implement
>> .set_dma_mask anyway, so we could possibly drop the references to it
>> altogether.
>>
>> Conversely, I suppose we could just implement said callback for
>> swiotlb_dma_ops and iommu_dma_ops with the parent_dma_mask-checking
>> function and drop the HAVE_ARCH_DMA_SET_MASK override instead. I'm not
>> sure which approach is preferable - the latter seems arguably cleaner in
>> isolation, but would also be less consistent with how the coherent mask
>> has to be handled. Ho hum.
>
> I mean, before patch is applied.
>
> In the current mainline codebase, arm64 iommu does setup dependent on
> [default] dma_mask, but does not anyhow react on dma mask change.
>
> I don't know much details about arm64 iommu, but from distant view this
> combination looks incorrect:
> - if behavior of this hardware should depend on dma_mask of device, then
> it should handle mask change,
> - if behavior of this hardware should not depend on dma_mask of device,
> then what for to pass size to it's setup?
Ah, right, I did indeed misunderstand. The IOMMU code is happy with the
DMA masks changing, since it always checks the appropriate one at the
point of IOVA allocation. The initial size given to
__iommu_setup_dma_ops() really just sets up a caching threshold in the
IOVA domain - if there is an explicitly-configured mask, then it's
generally good to set the limit based on that, but otherwise the default
32-bit limit is fine even if the driver subsequently alters the device's
mask(s) before making DMA API calls. Your patch won't actually change
any behaviour in this regard, as long as it still passes a 4GB size by
default.
Robin.
^ permalink raw reply
* [PATCH] KVM: arm64: Increase number of memslots to 512
From: linucherian at gmail.com @ 2017-01-11 16:52 UTC (permalink / raw)
To: linux-arm-kernel
From: Linu Cherian <linu.cherian@cavium.com>
Having only 32 memslots is a real constraint for the maximum number of
PCI devices that can be assigned to a single guest. Assuming each PCI
device/virtual function having two memory BAR regions, we could assign
only 15 devices/virtual functions to a guest.
So increase KVM_MEM_SLOTS_NUM to 512 as done in other archs like x86 and
powerpc. For this, KVM_USER_MEM_SLOTS has been changed to 508.
Signed-off-by: Linu Cherian <linu.cherian@cavium.com>
---
arch/arm/kvm/arm.c | 3 +++
arch/arm64/include/asm/kvm_host.h | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 8f92efa..a19389b 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -221,6 +221,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
case KVM_CAP_MAX_VCPUS:
r = KVM_MAX_VCPUS;
break;
+ case KVM_CAP_NR_MEMSLOTS:
+ r = KVM_USER_MEM_SLOTS;
+ break;
case KVM_CAP_MSI_DEVID:
if (!kvm)
r = -EINVAL;
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index e505038..88f017d 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -30,7 +30,7 @@
#define __KVM_HAVE_ARCH_INTC_INITIALIZED
-#define KVM_USER_MEM_SLOTS 32
+#define KVM_USER_MEM_SLOTS 508
#define KVM_PRIVATE_MEM_SLOTS 4
#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
#define KVM_HALT_POLL_NS_DEFAULT 500000
--
1.9.1
^ permalink raw reply related
* [PATCH v2 0/5] ARM: add module autoloading support for crypto modules
From: Ard Biesheuvel @ 2017-01-11 17:01 UTC (permalink / raw)
To: linux-arm-kernel
This series wires up the crypto modules that use the ARM 32-bit versions of
the ARMv8 Crypto Extensions to udev autoloading, by exposing the HWCAP2
feature bits via the CPU modalias. This is very similar to the arm64
implementation, with the notable exception that ARM has its CPU feature
definitions split across HWCAP and HWCAP2.
Given that the crypto feature bits are all exposed via HWCAP2, and considering
that there are currently no features exposed via HWCAP that are relevant to
udev module autoloading, exposing HWCAP2 only should be sufficient, at least
for now.
Changes since v1:
- rebase onto v4.10-rc2
Ard Biesheuvel (5):
ARM: wire up HWCAP2 feature bits to the CPU modalias
crypto: arm/aes-ce - enable module autoloading based on CPU feature
bits
crypto: arm/ghash-ce - enable module autoloading based on CPU feature
bits
crypto: arm/sha1-ce - enable module autoloading based on CPU feature
bits
crypto: arm/sha2-ce - enable module autoloading based on CPU feature
bits
arch/arm/Kconfig | 1 +
arch/arm/crypto/aes-ce-glue.c | 6 ++--
arch/arm/crypto/ghash-ce-glue.c | 6 ++--
arch/arm/crypto/sha1-ce-glue.c | 5 ++-
arch/arm/crypto/sha2-ce-glue.c | 5 ++-
arch/arm/include/asm/cpufeature.h | 32 ++++++++++++++++++++
6 files changed, 41 insertions(+), 14 deletions(-)
create mode 100644 arch/arm/include/asm/cpufeature.h
--
2.7.4
^ permalink raw reply
* [PATCH v2 1/5] ARM: wire up HWCAP2 feature bits to the CPU modalias
From: Ard Biesheuvel @ 2017-01-11 17:01 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1484154118-22139-1-git-send-email-ard.biesheuvel@linaro.org>
Wire up the generic support for exposing CPU feature bits via the
modalias in /sys/device/system/cpu. This allows udev to automatically
load modules for things like crypto algorithms that are implemented
using optional instructions.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm/Kconfig | 1 +
arch/arm/include/asm/cpufeature.h | 32 ++++++++++++++++++++
2 files changed, 33 insertions(+)
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 5fab553fd03a..3758466365ee 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -21,6 +21,7 @@ config ARM
select GENERIC_ALLOCATOR
select GENERIC_ATOMIC64 if (CPU_V7M || CPU_V6 || !CPU_32v6K || !AEABI)
select GENERIC_CLOCKEVENTS_BROADCAST if SMP
+ select GENERIC_CPU_AUTOPROBE
select GENERIC_EARLY_IOREMAP
select GENERIC_IDLE_POLL_SETUP
select GENERIC_IRQ_PROBE
diff --git a/arch/arm/include/asm/cpufeature.h b/arch/arm/include/asm/cpufeature.h
new file mode 100644
index 000000000000..134bd4c63d53
--- /dev/null
+++ b/arch/arm/include/asm/cpufeature.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __ASM_CPUFEATURE_H
+#define __ASM_CPUFEATURE_H
+
+#include <asm/hwcap.h>
+
+/*
+ * Due to the fact that ELF_HWCAP is a 32-bit type on ARM, and given the number
+ * of optional CPU features it defines, ARM's CPU capability bits have been
+ * distributed over separate elf_hwcap and elf_hwcap2 variables, each of which
+ * covers a subset of the available CPU features.
+ *
+ * Currently, only a few of those are suitable for automatic module loading
+ * (which is the primary use case of this facility) and those happen to be all
+ * covered by HWCAP2. So let's only expose those via the CPU modalias for now.
+ */
+#define MAX_CPU_FEATURES (8 * sizeof(elf_hwcap2))
+#define cpu_feature(x) ilog2(HWCAP2_ ## x)
+
+static inline bool cpu_have_feature(unsigned int num)
+{
+ return elf_hwcap2 & (1UL << num);
+}
+
+#endif
--
2.7.4
^ permalink raw reply related
* [PATCH v2 2/5] crypto: arm/aes-ce - enable module autoloading based on CPU feature bits
From: Ard Biesheuvel @ 2017-01-11 17:01 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1484154118-22139-1-git-send-email-ard.biesheuvel@linaro.org>
Make the module autoloadable by tying it to the CPU feature bit that
describes whether the optional instructions it relies on are implemented
by the current CPU.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm/crypto/aes-ce-glue.c | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c
index 8857531915bf..98c68ad101e5 100644
--- a/arch/arm/crypto/aes-ce-glue.c
+++ b/arch/arm/crypto/aes-ce-glue.c
@@ -14,6 +14,7 @@
#include <crypto/aes.h>
#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
+#include <linux/cpufeature.h>
#include <linux/module.h>
#include <crypto/xts.h>
@@ -428,9 +429,6 @@ static int __init aes_init(void)
int err;
int i;
- if (!(elf_hwcap2 & HWCAP2_AES))
- return -ENODEV;
-
err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
if (err)
return err;
@@ -454,5 +452,5 @@ static int __init aes_init(void)
return err;
}
-module_init(aes_init);
+module_cpu_feature_match(AES, aes_init);
module_exit(aes_exit);
--
2.7.4
^ permalink raw reply related
* [PATCH v2 3/5] crypto: arm/ghash-ce - enable module autoloading based on CPU feature bits
From: Ard Biesheuvel @ 2017-01-11 17:01 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1484154118-22139-1-git-send-email-ard.biesheuvel@linaro.org>
Make the module autoloadable by tying it to the CPU feature bit that
describes whether the optional instructions it relies on are implemented
by the current CPU.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm/crypto/ghash-ce-glue.c | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c
index 7546b3c02466..6bac8bea9f1e 100644
--- a/arch/arm/crypto/ghash-ce-glue.c
+++ b/arch/arm/crypto/ghash-ce-glue.c
@@ -15,6 +15,7 @@
#include <crypto/cryptd.h>
#include <crypto/internal/hash.h>
#include <crypto/gf128mul.h>
+#include <linux/cpufeature.h>
#include <linux/crypto.h>
#include <linux/module.h>
@@ -311,9 +312,6 @@ static int __init ghash_ce_mod_init(void)
{
int err;
- if (!(elf_hwcap2 & HWCAP2_PMULL))
- return -ENODEV;
-
err = crypto_register_shash(&ghash_alg);
if (err)
return err;
@@ -334,5 +332,5 @@ static void __exit ghash_ce_mod_exit(void)
crypto_unregister_shash(&ghash_alg);
}
-module_init(ghash_ce_mod_init);
+module_cpu_feature_match(PMULL, ghash_ce_mod_init);
module_exit(ghash_ce_mod_exit);
--
2.7.4
^ permalink raw reply related
* [PATCH v2 4/5] crypto: arm/sha1-ce - enable module autoloading based on CPU feature bits
From: Ard Biesheuvel @ 2017-01-11 17:01 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1484154118-22139-1-git-send-email-ard.biesheuvel@linaro.org>
Make the module autoloadable by tying it to the CPU feature bit that
describes whether the optional instructions it relies on are implemented
by the current CPU.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm/crypto/sha1-ce-glue.c | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c
index 80bc2fcd241a..555f72b5e659 100644
--- a/arch/arm/crypto/sha1-ce-glue.c
+++ b/arch/arm/crypto/sha1-ce-glue.c
@@ -11,6 +11,7 @@
#include <crypto/internal/hash.h>
#include <crypto/sha.h>
#include <crypto/sha1_base.h>
+#include <linux/cpufeature.h>
#include <linux/crypto.h>
#include <linux/module.h>
@@ -82,8 +83,6 @@ static struct shash_alg alg = {
static int __init sha1_ce_mod_init(void)
{
- if (!(elf_hwcap2 & HWCAP2_SHA1))
- return -ENODEV;
return crypto_register_shash(&alg);
}
@@ -92,5 +91,5 @@ static void __exit sha1_ce_mod_fini(void)
crypto_unregister_shash(&alg);
}
-module_init(sha1_ce_mod_init);
+module_cpu_feature_match(SHA1, sha1_ce_mod_init);
module_exit(sha1_ce_mod_fini);
--
2.7.4
^ permalink raw reply related
* [PATCH v2 5/5] crypto: arm/sha2-ce - enable module autoloading based on CPU feature bits
From: Ard Biesheuvel @ 2017-01-11 17:01 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1484154118-22139-1-git-send-email-ard.biesheuvel@linaro.org>
Make the module autoloadable by tying it to the CPU feature bit that
describes whether the optional instructions it relies on are implemented
by the current CPU.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm/crypto/sha2-ce-glue.c | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/arch/arm/crypto/sha2-ce-glue.c b/arch/arm/crypto/sha2-ce-glue.c
index 0755b2d657f3..df4dcef054ae 100644
--- a/arch/arm/crypto/sha2-ce-glue.c
+++ b/arch/arm/crypto/sha2-ce-glue.c
@@ -11,6 +11,7 @@
#include <crypto/internal/hash.h>
#include <crypto/sha.h>
#include <crypto/sha256_base.h>
+#include <linux/cpufeature.h>
#include <linux/crypto.h>
#include <linux/module.h>
@@ -100,8 +101,6 @@ static struct shash_alg algs[] = { {
static int __init sha2_ce_mod_init(void)
{
- if (!(elf_hwcap2 & HWCAP2_SHA2))
- return -ENODEV;
return crypto_register_shashes(algs, ARRAY_SIZE(algs));
}
@@ -110,5 +109,5 @@ static void __exit sha2_ce_mod_fini(void)
crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
}
-module_init(sha2_ce_mod_init);
+module_cpu_feature_match(SHA2, sha2_ce_mod_init);
module_exit(sha2_ce_mod_fini);
--
2.7.4
^ permalink raw reply related
* [PATCH] usb: dwc3 dwc3_exynos_probe() change goto labels to meaningful names
From: Javier Martinez Canillas @ 2017-01-11 17:06 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20170111164537.2981-1-shuahkh@osg.samsung.com>
Hello Shuah,
On 01/11/2017 01:45 PM, Shuah Khan wrote:
> Change goto labels to meaningful names from a series of errNs.
>
> Signed-off-by: Shuah Khan <shuahkh@osg.samsung.com>
> ---
> drivers/usb/dwc3/dwc3-exynos.c | 22 +++++++++++-----------
> 1 file changed, 11 insertions(+), 11 deletions(-)
>
> diff --git a/drivers/usb/dwc3/dwc3-exynos.c b/drivers/usb/dwc3/dwc3-exynos.c
> index f7421c2..ea50f74 100644
> --- a/drivers/usb/dwc3/dwc3-exynos.c
> +++ b/drivers/usb/dwc3/dwc3-exynos.c
> @@ -147,53 +147,53 @@ static int dwc3_exynos_probe(struct platform_device *pdev)
> exynos->vdd33 = devm_regulator_get(dev, "vdd33");
> if (IS_ERR(exynos->vdd33)) {
> ret = PTR_ERR(exynos->vdd33);
> - goto err2;
> + goto vdd33_err;
> }
I think it's better to use as labels what will be done in the error path
rather than what failed. IOW, "disable_axius_clk" or "disable_clks" are
more clear than vdd33_err for me, since can give you an idea of what will
happen in the error path.
But I don't have a strong opinion and your naming is an improvement over
the current one, so:
Reviewed-by: Javier Martinez Canillas <javier@osg.samsung.com>
Best regards,
--
Javier Martinez Canillas
Open Source Group
Samsung Research America
^ permalink raw reply
* [PATCH v5 00/12] mmc: Add support to Marvell Xenon SD Host Controller
From: Gregory CLEMENT @ 2017-01-11 17:19 UTC (permalink / raw)
To: linux-arm-kernel
Hello,
This the fifth version of the series adding support for the SDHCI
Xenon controller. It can be currently found on the Armada 37xx and the
Armada 7K/8K but will be also used in more Marvell SoC (and not only
the mvebu ones actually).
v4->v5:
- Remove the patch to export sdhci_execute_tuning(). It is already
exported in v4.10.
- Introduce a patch adding a missing clock for the sdhci controller
present on the CP master for A7K/A8K. There is no build dependency
but obviously this patch is need to use the sdhci controller present
on the CP part.
- Adjust Xenon return setup, to avoid being overwritten by
sdhci_add_host().
- Change Xenon register definition prefix to "XENON_".
- Fix typos in Xenon driver and dt-binding docs.
- Change compatible string "marvell,armada-7000-sdhci" to
"marvell,armada-8k-sdhci". Actually the Armada 7K SoCs are a subset
of the Armada 8K SoCs. Moreover, the use of the '000' is not
consistent with all the other compatible string already used for the
Armada 7K/8K family.
- Added the Tested-by from Russell King on an Armada 8K based board.
v3 -> v4:
For this version a few change have been done:
- fixes 2 bug reported by kbuild-bot
- remove extra of_node_put()
- convert 0 in false for function returning boolean
- add a device tree node for the sdhci controller present on the CP
master for A7K/A8K. It also led to rename the sdhci0 node on AP to
ap_sdhci0 to make a distinction with the one present on CP master.
v2 -> v3
I think that now most (if not all) the remarks had been taking into
account since the second version. According to Ziji Hu, here are the
following changes:
" Changes in V3:
Adjust and improve Xenon DT bindings. Move some caps setting from driver into
DT. Use mmc-card sub-node to represent eMMC type.
Remove PHY Sampling Fixed Delay Line scan in lower speed mode.
Improve Xenon probe and ->init_card() functions.
Export sdhci_enable_sdio_irq() and implement own SDIO IRQ control.
Split PHY patch into two smaller patches.
Temporarily remove AXI clock before its implementation is improved."
Besides this changes I also
- Removed the sdhci-xenon-phy.h and moved its content in the
shc-xenon-phy.c file.
- Fixed the tuning-count usage
- Managed the error case for clk_prepare_enable
For the record the change from v1 was:
" Changes in V2:
rebase on v4.9-rc2.
Re-write Xenon bindings. Ajust Xenon DT property naming.
Add a new DT property to indicate eMMC card type, instead of using
variable card_candidate.
Clear quirks SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12 in Xenon platform data
Add support to HS400 retuning."
Thanks,
Gregory
Gregory CLEMENT (3):
arm64: dts: marvell: add eMMC support for Armada 37xx
arm64: dts: marvell: add sdhci support for Armada 7K/8K
arm64: configs: enable SDHCI driver for Xenon
Hu Ziji (8):
mmc: sdhci: Export sdhci_set_ios() from sdhci.c
mmc: sdhci: Export sdhci_start_signal_voltage_switch() in sdhci.c
mmc: sdhci: Export sdhci_enable_sdio_irq() from sdhci.c
dt: bindings: Add bindings for Marvell Xenon SD Host Controller
mmc: sdhci-xenon: Add Marvell Xenon SDHC core functionality
mmc: sdhci-xenon: Add support to PHYs of Marvell Xenon SDHC.
mmc: sdhci-xenon: Add SoC PHY PAD voltage control
MAINTAINERS: add entry for Marvell Xenon MMC Host Controller drivers
Konstantin Porotchkin (1):
clk: apn806: Add eMMC clock to system controller driver
Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt | 197 ++-
MAINTAINERS | 7 +-
arch/arm64/boot/dts/marvell/armada-3720-db.dts | 16 +-
arch/arm64/boot/dts/marvell/armada-37xx.dtsi | 11 +-
arch/arm64/boot/dts/marvell/armada-7040-db.dts | 14 +-
arch/arm64/boot/dts/marvell/armada-ap806.dtsi | 13 +-
arch/arm64/boot/dts/marvell/armada-cp110-master.dtsi | 11 +-
arch/arm64/configs/defconfig | 1 +-
drivers/clk/mvebu/ap806-system-controller.c | 15 +-
drivers/mmc/host/Kconfig | 9 +-
drivers/mmc/host/Makefile | 3 +-
drivers/mmc/host/sdhci-xenon-phy.c | 904 +++++++-
drivers/mmc/host/sdhci-xenon.c | 634 +++++-
drivers/mmc/host/sdhci-xenon.h | 109 +-
drivers/mmc/host/sdhci.c | 11 +-
drivers/mmc/host/sdhci.h | 4 +-
16 files changed, 1953 insertions(+), 6 deletions(-)
create mode 100644 Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt
create mode 100644 drivers/mmc/host/sdhci-xenon-phy.c
create mode 100644 drivers/mmc/host/sdhci-xenon.c
create mode 100644 drivers/mmc/host/sdhci-xenon.h
base-commit: 0c744ea4f77d72b3dcebb7a8f2684633ec79be88
--
git-series 0.9.1
^ permalink raw reply
* [PATCH v5 01/12] clk: apn806: Add eMMC clock to system controller driver
From: Gregory CLEMENT @ 2017-01-11 17:19 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <cover.8527229dae6c124f24c5e637430c2cdc86f80392.1484154449.git-series.gregory.clement@free-electrons.com>
From: Konstantin Porotchkin <kostap@marvell.com>
Add fixed clock of 400MHz to system controller driver. This clock is
used as SD/eMMC clock source.
Signed-off-by: Konstantin Porotchkin <kostap@marvell.com>
Reviewed-by: Omri Itach <omrii@marvell.com>
Reviewed-by: Hanna Hawa <hannah@marvell.com>
[fixed up conflicts, added error handling --rmk]
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
Signed-off-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
---
arch/arm64/boot/dts/marvell/armada-ap806.dtsi | 3 ++-
drivers/clk/mvebu/ap806-system-controller.c | 15 ++++++++++++++-
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/boot/dts/marvell/armada-ap806.dtsi b/arch/arm64/boot/dts/marvell/armada-ap806.dtsi
index a749ba2edec4..5019c8f4acd0 100644
--- a/arch/arm64/boot/dts/marvell/armada-ap806.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-ap806.dtsi
@@ -235,7 +235,8 @@
#clock-cells = <1>;
clock-output-names = "ap-cpu-cluster-0",
"ap-cpu-cluster-1",
- "ap-fixed", "ap-mss";
+ "ap-fixed", "ap-mss",
+ "ap-emmc";
reg = <0x6f4000 0x1000>;
};
};
diff --git a/drivers/clk/mvebu/ap806-system-controller.c b/drivers/clk/mvebu/ap806-system-controller.c
index 8181b919f062..0a785076b0c6 100644
--- a/drivers/clk/mvebu/ap806-system-controller.c
+++ b/drivers/clk/mvebu/ap806-system-controller.c
@@ -23,7 +23,7 @@
#define AP806_SAR_REG 0x400
#define AP806_SAR_CLKFREQ_MODE_MASK 0x1f
-#define AP806_CLK_NUM 4
+#define AP806_CLK_NUM 5
static struct clk *ap806_clks[AP806_CLK_NUM];
@@ -117,6 +117,17 @@ static int ap806_syscon_clk_probe(struct platform_device *pdev)
goto fail3;
}
+ /* eMMC Clock is fixed clock divided by 3 */
+ of_property_read_string_index(np, "clock-output-names",
+ 4, &name);
+ ap806_clks[4] = clk_register_fixed_factor(NULL, name, fixedclk_name,
+ 0, 1, 3);
+ if (IS_ERR(ap806_clks[4])) {
+ ret = PTR_ERR(ap806_clks[4]);
+ goto fail4;
+ }
+
+ of_clk_add_provider(np, of_clk_src_onecell_get, &ap806_clk_data);
ret = of_clk_add_provider(np, of_clk_src_onecell_get, &ap806_clk_data);
if (ret)
goto fail_clk_add;
@@ -124,6 +135,8 @@ static int ap806_syscon_clk_probe(struct platform_device *pdev)
return 0;
fail_clk_add:
+ clk_unregister_fixed_factor(ap806_clks[4]);
+fail4:
clk_unregister_fixed_factor(ap806_clks[3]);
fail3:
clk_unregister_fixed_rate(ap806_clks[2]);
--
git-series 0.9.1
^ permalink raw reply related
* [PATCH v5 02/12] mmc: sdhci: Export sdhci_set_ios() from sdhci.c
From: Gregory CLEMENT @ 2017-01-11 17:19 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <cover.8527229dae6c124f24c5e637430c2cdc86f80392.1484154449.git-series.gregory.clement@free-electrons.com>
From: Hu Ziji <huziji@marvell.com>
Export sdhci_set_ios() in sdhci.c.
Thus vendor sdhci driver can implement its own set_ios() routine.
Signed-off-by: Hu Ziji <huziji@marvell.com>
Signed-off-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
---
drivers/mmc/host/sdhci.c | 3 ++-
drivers/mmc/host/sdhci.h | 1 +
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c
index 23909804ffb8..0e9ed0fadff9 100644
--- a/drivers/mmc/host/sdhci.c
+++ b/drivers/mmc/host/sdhci.c
@@ -1570,7 +1570,7 @@ void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
}
EXPORT_SYMBOL_GPL(sdhci_set_uhs_signaling);
-static void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
+void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
{
struct sdhci_host *host = mmc_priv(mmc);
unsigned long flags;
@@ -1730,6 +1730,7 @@ static void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
mmiowb();
spin_unlock_irqrestore(&host->lock, flags);
}
+EXPORT_SYMBOL_GPL(sdhci_set_ios);
static int sdhci_get_cd(struct mmc_host *mmc)
{
diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h
index 0b66f210ae82..3563c0c384ca 100644
--- a/drivers/mmc/host/sdhci.h
+++ b/drivers/mmc/host/sdhci.h
@@ -691,6 +691,7 @@ void sdhci_set_bus_width(struct sdhci_host *host, int width);
void sdhci_reset(struct sdhci_host *host, u8 mask);
void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing);
int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode);
+void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios);
#ifdef CONFIG_PM
extern int sdhci_suspend_host(struct sdhci_host *host);
--
git-series 0.9.1
^ permalink raw reply related
* [PATCH v5 03/12] mmc: sdhci: Export sdhci_start_signal_voltage_switch() in sdhci.c
From: Gregory CLEMENT @ 2017-01-11 17:19 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <cover.8527229dae6c124f24c5e637430c2cdc86f80392.1484154449.git-series.gregory.clement@free-electrons.com>
From: Hu Ziji <huziji@marvell.com>
Export sdhci_start_signal_voltage_switch() from sdhci.c.
Thus vendor sdhci driver can implement its own signal voltage
switch routine.
Signed-off-by: Hu Ziji <huziji@marvell.com>
Signed-off-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
---
drivers/mmc/host/sdhci.c | 5 +++--
drivers/mmc/host/sdhci.h | 2 ++
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c
index 0e9ed0fadff9..d2338dc362f4 100644
--- a/drivers/mmc/host/sdhci.c
+++ b/drivers/mmc/host/sdhci.c
@@ -1839,8 +1839,8 @@ static void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable)
spin_unlock_irqrestore(&host->lock, flags);
}
-static int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
- struct mmc_ios *ios)
+int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
+ struct mmc_ios *ios)
{
struct sdhci_host *host = mmc_priv(mmc);
u16 ctrl;
@@ -1932,6 +1932,7 @@ static int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
return 0;
}
}
+EXPORT_SYMBOL_GPL(sdhci_start_signal_voltage_switch);
static int sdhci_card_busy(struct mmc_host *mmc)
{
diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h
index 3563c0c384ca..e4a07543a12b 100644
--- a/drivers/mmc/host/sdhci.h
+++ b/drivers/mmc/host/sdhci.h
@@ -692,6 +692,8 @@ void sdhci_reset(struct sdhci_host *host, u8 mask);
void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing);
int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode);
void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios);
+int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
+ struct mmc_ios *ios);
#ifdef CONFIG_PM
extern int sdhci_suspend_host(struct sdhci_host *host);
--
git-series 0.9.1
^ permalink raw reply related
* [PATCH v5 04/12] mmc: sdhci: Export sdhci_enable_sdio_irq() from sdhci.c
From: Gregory CLEMENT @ 2017-01-11 17:19 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <cover.8527229dae6c124f24c5e637430c2cdc86f80392.1484154449.git-series.gregory.clement@free-electrons.com>
From: Hu Ziji <huziji@marvell.com>
Export sdhci_enable_sdio_irq() from sdhci.c.
Thus vendor SDHC driver can implement its specific SDIO irq
control.
Signed-off-by: Hu Ziji <huziji@marvell.com>
Signed-off-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
---
drivers/mmc/host/sdhci.c | 3 ++-
drivers/mmc/host/sdhci.h | 1 +
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c
index d2338dc362f4..9a4fda81ff81 100644
--- a/drivers/mmc/host/sdhci.c
+++ b/drivers/mmc/host/sdhci.c
@@ -1824,7 +1824,7 @@ static void sdhci_enable_sdio_irq_nolock(struct sdhci_host *host, int enable)
}
}
-static void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable)
+void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable)
{
struct sdhci_host *host = mmc_priv(mmc);
unsigned long flags;
@@ -1838,6 +1838,7 @@ static void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable)
sdhci_enable_sdio_irq_nolock(host, enable);
spin_unlock_irqrestore(&host->lock, flags);
}
+EXPORT_SYMBOL_GPL(sdhci_enable_sdio_irq);
int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
struct mmc_ios *ios)
diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h
index e4a07543a12b..609783364941 100644
--- a/drivers/mmc/host/sdhci.h
+++ b/drivers/mmc/host/sdhci.h
@@ -694,6 +694,7 @@ int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode);
void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios);
int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
struct mmc_ios *ios);
+void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable);
#ifdef CONFIG_PM
extern int sdhci_suspend_host(struct sdhci_host *host);
--
git-series 0.9.1
^ permalink raw reply related
* [PATCH v5 05/12] dt: bindings: Add bindings for Marvell Xenon SD Host Controller
From: Gregory CLEMENT @ 2017-01-11 17:19 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <cover.8527229dae6c124f24c5e637430c2cdc86f80392.1484154449.git-series.gregory.clement@free-electrons.com>
From: Hu Ziji <huziji@marvell.com>
Marvell Xenon SDHC can support eMMC/SD/SDIO.
Add Xenon-specific properties.
Also add properties for Xenon PHY setting.
Signed-off-by: Hu Ziji <huziji@marvell.com>
Signed-off-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
---
Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt | 197 +++++++-
1 file changed, 197 insertions(+)
create mode 100644 Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt
diff --git a/Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt b/Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt
new file mode 100644
index 000000000000..a3876d2cc616
--- /dev/null
+++ b/Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt
@@ -0,0 +1,197 @@
+Marvell Xenon SDHCI Controller device tree bindings
+This file documents differences between the core mmc properties
+described by mmc.txt and the properties used by the Xenon implementation.
+
+Multiple SDHCs might be put into a single Xenon IP, to save size and cost.
+Each SDHC is independent and owns independent resources, such as register sets,
+clock and PHY.
+Each SDHC should have an independent device tree node.
+
+Required Properties:
+- compatible: should be one of the following
+ - "marvell,armada-3700-sdhci": For controllers on Armada-3700 SoC.
+ Must provide a second register area and marvell,pad-type.
+ - "marvell,armada-8k-sdhci": For controllers on Armada 7K/8K SoC
+
+- clocks:
+ Array of clocks required for SDHC.
+ Require at least input clock for Xenon IP core.
+
+- clock-names:
+ Array of names corresponding to clocks property.
+ The input clock for Xenon IP core should be named as "core".
+
+- reg:
+ * For "marvell,armada-3700-sdhci", two register areas.
+ The first one for Xenon IP register. The second one for the Armada 3700 SoC
+ PHY PAD Voltage Control register.
+ Please follow the examples with compatible "marvell,armada-3700-sdhci"
+ in below.
+ Please also check property marvell,pad-type in below.
+
+ * For other compatible strings, one register area for Xenon IP.
+
+Optional Properties:
+- mmccard:
+ mmccard child node must be provided when current SDHC is for eMMC.
+ Xenon SDHC often can support both SD and eMMC. This child node indicates that
+ current SDHC is for eMMC card. Thus Xenon eMMC specific configuration and
+ operations can be enabled prior to eMMC init sequence.
+ Please refer to Documentation/devicetree/bindings/mmc/mmc-card.txt.
+ This child node should not be set if current Xenon SDHC is for SD/SDIO.
+
+- bus-width:
+ When 8-bit data bus width is in use for eMMC, this property should be
+ explicitly provided and set as 8.
+ It is optional when data bus width is 4-bit or 1-bit.
+
+- mmc-ddr-1_8v:
+ Select this property when eMMC HS DDR is supported on SDHC side.
+
+- mmc-hs400-1_8v:
+ Select this property when eMMC HS400 is supported on SDHC side.
+
+- no-1-8-v:
+ Select this property when 1.8V signaling voltage supply is unavailable.
+ When this property is enabled, both mmc-ddr-1_8v and mmc-hs400-1_8v should be
+ cleared.
+
+- marvell,xenon-sdhc-id:
+ Indicate the corresponding bit index of current SDHC in
+ SDHC System Operation Control Register Bit[7:0].
+ Set/clear the corresponding bit to enable/disable current SDHC.
+ If Xenon IP contains only one SDHC, this property is optional.
+
+- marvell,xenon-phy-type:
+ Xenon support mutilple types of PHYs.
+ To select eMMC 5.1 PHY, set:
+ marvell,xenon-phy-type = "emmc 5.1 phy"
+ eMMC 5.1 PHY is the default choice if this property is not provided.
+ To select eMMC 5.0 PHY, set:
+ marvell,xenon-phy-type = "emmc 5.0 phy"
+
+ All those types of PHYs can support eMMC, SD and SDIO.
+ Please note that this property only presents the type of PHY.
+ It doesn't stand for the entire SDHC type or property.
+ For example, "emmc 5.1 phy" doesn't mean that this Xenon SDHC only supports
+ eMMC 5.1.
+
+- marvell,xenon-phy-znr:
+ Set PHY ZNR value.
+ Only available for eMMC PHY 5.1 and eMMC PHY 5.0.
+ Valid range = [0:0x1F].
+ ZNR is set as 0xF by default if this property is not provided.
+
+- marvell,xenon-phy-zpr:
+ Set PHY ZPR value.
+ Only available for eMMC PHY 5.1 and eMMC PHY 5.0.
+ Valid range = [0:0x1F].
+ ZPR is set as 0xF by default if this property is not provided.
+
+- marvell,xenon-phy-nr-success-tun:
+ Set the number of required consecutive successful sampling points used to
+ identify a valid sampling window, in tuning process.
+ Valid range = [1:7].
+ Set as 0x4 by default if this property is not provided.
+
+- marvell,xenon-phy-tun-step-divider:
+ Set the divider for calculating TUN_STEP.
+ Set as 64 by default if this property is not provided.
+
+- marvell,xenon-phy-slow-mode:
+ If this property is selected, transfers will bypass PHY.
+ Only available when bus frequency lower than 55MHz in SDR mde.
+ Disabled by default. Please only try this property if timing issues always
+ occur with PHY enabled in eMMC HS SDR, SD SDR12, SD SDR25, SD SDR50 mode.
+
+- marvell,xenon-tun-count:
+ Xenon SDHC SoC usually doesn't provide re-tuning counter in
+ Capabilities Register 3 Bit[11:8].
+ This property provides the re-tuning counter.
+ If this property is not set, default re-tuning counter will
+ be set as 0x9 in driver.
+
+- marvell,pad-type:
+ Type of Armada 3700 SoC PHY PAD Voltage Controller register.
+ Only valid when "marvell,armada-3700-sdhci" is selected.
+ Two types: "sd" and "fixed-1-8v".
+ If "sd" is slected, SoC PHY PAD is set as 3.3V at the beginning and is
+ switched to 1.8V when SD in UHS-I.
+ If "fixed-1-8v" is slected, SoC PHY PAD is fixed 1.8V, such as for eMMC.
+ Please follow the examples with compatible "marvell,armada-3700-sdhci"
+ in below.
+
+Example:
+- For eMMC:
+
+ sdhci at aa0000 {
+ compatible = "marvell,armada-8k-sdhci";
+ reg = <0xaa0000 0x1000>;
+ interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>
+ clocks = <&emmc_clk>;
+ clock-names = "core";
+ bus-width = <8>;
+ mmc-ddr-1_8v;
+ mmc-hs400-1_8v;
+ marvell,xenon-sdhc-id = <0>;
+ marvell,xenon-phy-type = "emmc 5.1 phy";
+ marvell,xenon-tun-count = <11>;
+
+ #address-cells = <1>;
+ #size-cells = <0>;
+ mmccard: mmccard at 0 {
+ compatible = "mmc-card";
+ reg = <0>;
+ };
+ };
+
+- For SD/SDIO:
+
+ sdhci at ab0000 {
+ compatible = "marvell,armada-8k-sdhci";
+ reg = <0xab0000 0x1000>;
+ interrupts = <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>
+ vqmmc-supply = <&sd_regulator>;
+ clocks = <&sdclk>;
+ clock-names = "core";
+ bus-width = <4>;
+ marvell,xenon-tun-count = <9>;
+ };
+
+- For eMMC with compatible "marvell,armada-3700-sdhci":
+
+ sdhci at aa0000 {
+ compatible = "marvell,armada-3700-sdhci";
+ reg = <0xaa0000 0x1000>,
+ <phy_addr 0x4>;
+ interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>
+ clocks = <&emmcclk>;
+ clock-names = "core";
+ bus-width = <8>;
+ mmc-ddr-1_8v;
+ mmc-hs400-1_8v;
+
+ marvell,pad-type = "fixed-1-8v";
+
+ #address-cells = <1>;
+ #size-cells = <0>;
+ mmccard: mmccard at 0 {
+ compatible = "mmc-card";
+ reg = <0>;
+ };
+ };
+
+- For SD/SDIO with compatible "marvell,armada-3700-sdhci":
+
+ sdhci at ab0000 {
+ compatible = "marvell,armada-3700-sdhci";
+ reg = <0xab0000 0x1000>,
+ <phy_addr 0x4>;
+ interrupts = <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>
+ vqmmc-supply = <&sd_regulator>;
+ clocks = <&sdclk>;
+ clock-names = "core";
+ bus-width = <4>;
+
+ marvell,pad-type = "sd";
+ };
--
git-series 0.9.1
^ permalink raw reply related
* [PATCH v5 06/12] mmc: sdhci-xenon: Add Marvell Xenon SDHC core functionality
From: Gregory CLEMENT @ 2017-01-11 17:19 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <cover.8527229dae6c124f24c5e637430c2cdc86f80392.1484154449.git-series.gregory.clement@free-electrons.com>
From: Hu Ziji <huziji@marvell.com>
Add Xenon eMMC/SD/SDIO host controller core functionality.
Add Xenon specific intialization process.
Add Xenon specific mmc_host_ops APIs.
Add Xenon specific register definitions.
Add CONFIG_MMC_SDHCI_XENON support in drivers/mmc/host/Kconfig.
Marvell Xenon SDHC conforms to SD Physical Layer Specification
Version 3.01 and is designed according to the guidelines provided
in the SD Host Controller Standard Specification Version 3.00.
Signed-off-by: Hu Ziji <huziji@marvell.com>
Tested-by: Russell King <rmk+kernel@armlinux.org.uk>
Signed-off-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
---
drivers/mmc/host/Kconfig | 9 +-
drivers/mmc/host/Makefile | 3 +-
drivers/mmc/host/sdhci-xenon.c | 631 ++++++++++++++++++++++++++++++++++-
drivers/mmc/host/sdhci-xenon.h | 70 ++++-
4 files changed, 713 insertions(+)
create mode 100644 drivers/mmc/host/sdhci-xenon.c
create mode 100644 drivers/mmc/host/sdhci-xenon.h
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index 2eb97014dc3f..8d2d08de14a0 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -819,3 +819,12 @@ config MMC_SDHCI_BRCMSTB
Broadcom STB SoCs.
If unsure, say Y.
+
+config MMC_SDHCI_XENON
+ tristate "Marvell Xenon eMMC/SD/SDIO SDHCI driver"
+ depends on MMC_SDHCI && MMC_SDHCI_PLTFM
+ help
+ This selects Marvell Xenon eMMC/SD/SDIO SDHCI.
+ If you have a machine with integrated Marvell Xenon SDHC IP,
+ say Y or M here.
+ If unsure, say N.
diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile
index ccc9c4cba154..b0a2ab4b256e 100644
--- a/drivers/mmc/host/Makefile
+++ b/drivers/mmc/host/Makefile
@@ -82,3 +82,6 @@ obj-$(CONFIG_MMC_SDHCI_BRCMSTB) += sdhci-brcmstb.o
ifeq ($(CONFIG_CB710_DEBUG),y)
CFLAGS-cb710-mmc += -DDEBUG
endif
+
+obj-$(CONFIG_MMC_SDHCI_XENON) += sdhci-xenon-driver.o
+sdhci-xenon-driver-y += sdhci-xenon.o
diff --git a/drivers/mmc/host/sdhci-xenon.c b/drivers/mmc/host/sdhci-xenon.c
new file mode 100644
index 000000000000..0e0c60892f72
--- /dev/null
+++ b/drivers/mmc/host/sdhci-xenon.c
@@ -0,0 +1,631 @@
+/*
+ * Driver for Marvell Xenon SDHC as a platform device
+ *
+ * Copyright (C) 2016 Marvell, All Rights Reserved.
+ *
+ * Author: Hu Ziji <huziji@marvell.com>
+ * Date: 2016-8-24
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * Inspired by Jisheng Zhang <jszhang@marvell.com>
+ * Special thanks to Video BG4 project team.
+ */
+
+#include <linux/delay.h>
+#include <linux/module.h>
+#include <linux/of.h>
+
+#include "sdhci-pltfm.h"
+#include "sdhci-xenon.h"
+
+static int enable_xenon_internal_clk(struct sdhci_host *host)
+{
+ u32 reg;
+ u8 timeout;
+
+ reg = sdhci_readl(host, SDHCI_CLOCK_CONTROL);
+ reg |= SDHCI_CLOCK_INT_EN;
+ sdhci_writel(host, reg, SDHCI_CLOCK_CONTROL);
+ /* Wait max 20 ms */
+ timeout = 20;
+ while (!((reg = sdhci_readw(host, SDHCI_CLOCK_CONTROL))
+ & SDHCI_CLOCK_INT_STABLE)) {
+ if (timeout == 0) {
+ pr_err("%s: Internal clock never stabilised.\n",
+ mmc_hostname(host->mmc));
+ return -ETIMEDOUT;
+ }
+ timeout--;
+ mdelay(1);
+ }
+
+ return 0;
+}
+
+/* Set SDCLK-off-while-idle */
+static void xenon_set_sdclk_off_idle(struct sdhci_host *host,
+ unsigned char sdhc_id, bool enable)
+{
+ u32 reg;
+ u32 mask;
+
+ reg = sdhci_readl(host, XENON_SYS_OP_CTRL);
+ /* Get the bit shift basing on the SDHC index */
+ mask = (0x1 << (XENON_SDCLK_IDLEOFF_ENABLE_SHIFT + sdhc_id));
+ if (enable)
+ reg |= mask;
+ else
+ reg &= ~mask;
+
+ sdhci_writel(host, reg, XENON_SYS_OP_CTRL);
+}
+
+/* Enable/Disable the Auto Clock Gating function */
+static void xenon_set_acg(struct sdhci_host *host, bool enable)
+{
+ u32 reg;
+
+ reg = sdhci_readl(host, XENON_SYS_OP_CTRL);
+ if (enable)
+ reg &= ~XENON_AUTO_CLKGATE_DISABLE_MASK;
+ else
+ reg |= XENON_AUTO_CLKGATE_DISABLE_MASK;
+ sdhci_writel(host, reg, XENON_SYS_OP_CTRL);
+}
+
+/* Enable this SDHC */
+static void xenon_enable_sdhc(struct sdhci_host *host,
+ unsigned char sdhc_id)
+{
+ u32 reg;
+
+ reg = sdhci_readl(host, XENON_SYS_OP_CTRL);
+ reg |= (BIT(sdhc_id) << XENON_SLOT_ENABLE_SHIFT);
+ sdhci_writel(host, reg, XENON_SYS_OP_CTRL);
+
+ /*
+ * Manually set the flag which all the card types require,
+ * including SD, eMMC, SDIO
+ */
+ host->mmc->caps |= MMC_CAP_WAIT_WHILE_BUSY;
+}
+
+/* Disable this SDHC */
+static void xenon_disable_sdhc(struct sdhci_host *host,
+ unsigned char sdhc_id)
+{
+ u32 reg;
+
+ reg = sdhci_readl(host, XENON_SYS_OP_CTRL);
+ reg &= ~(BIT(sdhc_id) << XENON_SLOT_ENABLE_SHIFT);
+ sdhci_writel(host, reg, XENON_SYS_OP_CTRL);
+}
+
+/* Enable Parallel Transfer Mode */
+static void xenon_enable_sdhc_parallel_tran(struct sdhci_host *host,
+ unsigned char sdhc_id)
+{
+ u32 reg;
+
+ reg = sdhci_readl(host, XENON_SYS_EXT_OP_CTRL);
+ reg |= BIT(sdhc_id);
+ sdhci_writel(host, reg, XENON_SYS_EXT_OP_CTRL);
+}
+
+/* Mask command conflict error */
+static void xenon_mask_cmd_conflict_err(struct sdhci_host *host)
+{
+ u32 reg;
+
+ reg = sdhci_readl(host, XENON_SYS_EXT_OP_CTRL);
+ reg |= XENON_MASK_CMD_CONFLICT_ERR;
+ sdhci_writel(host, reg, XENON_SYS_EXT_OP_CTRL);
+}
+
+static void xenon_sdhc_retune_setup(struct sdhci_host *host)
+{
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ u32 reg;
+
+ /* Disable the Re-Tuning Request functionality */
+ reg = sdhci_readl(host, XENON_SLOT_RETUNING_REQ_CTRL);
+ reg &= ~XENON_RETUNING_COMPATIBLE;
+ sdhci_writel(host, reg, XENON_SLOT_RETUNING_REQ_CTRL);
+
+ /* Disable the Re-tuning Interrupt */
+ reg = sdhci_readl(host, SDHCI_SIGNAL_ENABLE);
+ reg &= ~SDHCI_INT_RETUNE;
+ sdhci_writel(host, reg, SDHCI_SIGNAL_ENABLE);
+ reg = sdhci_readl(host, SDHCI_INT_ENABLE);
+ reg &= ~SDHCI_INT_RETUNE;
+ sdhci_writel(host, reg, SDHCI_INT_ENABLE);
+
+ /* Force to use Tuning Mode 1 */
+ host->tuning_mode = SDHCI_TUNING_MODE_1;
+ /* Set re-tuning period */
+ host->tuning_count = 1 << (priv->tuning_count - 1);
+}
+
+/*
+ * Operations inside struct sdhci_ops
+ */
+/* Recover the Register Setting cleared during SOFTWARE_RESET_ALL */
+static void sdhci_xenon_reset_exit(struct sdhci_host *host,
+ unsigned char sdhc_id, u8 mask)
+{
+ /* Only SOFTWARE RESET ALL will clear the register setting */
+ if (!(mask & SDHCI_RESET_ALL))
+ return;
+
+ /* Disable tuning request and auto-retuning again */
+ xenon_sdhc_retune_setup(host);
+
+ xenon_set_acg(host, true);
+
+ xenon_set_sdclk_off_idle(host, sdhc_id, false);
+}
+
+static void sdhci_xenon_reset(struct sdhci_host *host, u8 mask)
+{
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+
+ sdhci_reset(host, mask);
+ sdhci_xenon_reset_exit(host, priv->sdhc_id, mask);
+}
+
+/*
+ * Xenon defines different values for HS200 and HS400
+ * in Host_Control_2
+ */
+static void xenon_set_uhs_signaling(struct sdhci_host *host,
+ unsigned int timing)
+{
+ u16 ctrl_2;
+
+ ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
+ /* Select Bus Speed Mode for host */
+ ctrl_2 &= ~SDHCI_CTRL_UHS_MASK;
+ if (timing == MMC_TIMING_MMC_HS200)
+ ctrl_2 |= XENON_CTRL_HS200;
+ else if (timing == MMC_TIMING_UHS_SDR104)
+ ctrl_2 |= SDHCI_CTRL_UHS_SDR104;
+ else if (timing == MMC_TIMING_UHS_SDR12)
+ ctrl_2 |= SDHCI_CTRL_UHS_SDR12;
+ else if (timing == MMC_TIMING_UHS_SDR25)
+ ctrl_2 |= SDHCI_CTRL_UHS_SDR25;
+ else if (timing == MMC_TIMING_UHS_SDR50)
+ ctrl_2 |= SDHCI_CTRL_UHS_SDR50;
+ else if ((timing == MMC_TIMING_UHS_DDR50) ||
+ (timing == MMC_TIMING_MMC_DDR52))
+ ctrl_2 |= SDHCI_CTRL_UHS_DDR50;
+ else if (timing == MMC_TIMING_MMC_HS400)
+ ctrl_2 |= XENON_CTRL_HS400;
+ sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
+}
+
+static const struct sdhci_ops sdhci_xenon_ops = {
+ .set_clock = sdhci_set_clock,
+ .set_bus_width = sdhci_set_bus_width,
+ .reset = sdhci_xenon_reset,
+ .set_uhs_signaling = xenon_set_uhs_signaling,
+ .get_max_clock = sdhci_pltfm_clk_get_max_clock,
+};
+
+static const struct sdhci_pltfm_data sdhci_xenon_pdata = {
+ .ops = &sdhci_xenon_ops,
+ .quirks = SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC |
+ SDHCI_QUIRK_NO_SIMULT_VDD_AND_POWER |
+ SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
+};
+
+/*
+ * Xenon Specific Operations in mmc_host_ops
+ */
+static void xenon_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
+{
+ struct sdhci_host *host = mmc_priv(mmc);
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ unsigned long flags;
+ u32 reg;
+
+ /*
+ * HS400/HS200/eMMC HS doesn't have Preset Value register.
+ * However, sdhci_set_ios will read HS400/HS200 Preset register.
+ * Disable Preset Value register for HS400/HS200.
+ * eMMC HS with preset_enabled set will trigger a bug in
+ * get_preset_value().
+ */
+ spin_lock_irqsave(&host->lock, flags);
+ if ((ios->timing == MMC_TIMING_MMC_HS400) ||
+ (ios->timing == MMC_TIMING_MMC_HS200) ||
+ (ios->timing == MMC_TIMING_MMC_HS)) {
+ host->preset_enabled = false;
+ host->quirks2 |= SDHCI_QUIRK2_PRESET_VALUE_BROKEN;
+
+ reg = sdhci_readw(host, SDHCI_HOST_CONTROL2);
+ reg &= ~SDHCI_CTRL_PRESET_VAL_ENABLE;
+ sdhci_writew(host, reg, SDHCI_HOST_CONTROL2);
+ } else {
+ host->quirks2 &= ~SDHCI_QUIRK2_PRESET_VALUE_BROKEN;
+ }
+ spin_unlock_irqrestore(&host->lock, flags);
+
+ sdhci_set_ios(mmc, ios);
+
+ if (host->clock > XENON_DEFAULT_SDCLK_FREQ) {
+ spin_lock_irqsave(&host->lock, flags);
+ xenon_set_sdclk_off_idle(host, priv->sdhc_id, true);
+ spin_unlock_irqrestore(&host->lock, flags);
+ }
+}
+
+static int xenon_emmc_signal_voltage_switch(struct mmc_host *mmc,
+ struct mmc_ios *ios)
+{
+ unsigned char voltage = ios->signal_voltage;
+ struct sdhci_host *host = mmc_priv(mmc);
+ unsigned char voltage_code;
+ u32 ctrl;
+
+ if ((voltage == MMC_SIGNAL_VOLTAGE_330) ||
+ (voltage == MMC_SIGNAL_VOLTAGE_180)) {
+ if (voltage == MMC_SIGNAL_VOLTAGE_330)
+ voltage_code = XENON_EMMC_VCCQ_3_3V;
+ else if (voltage == MMC_SIGNAL_VOLTAGE_180)
+ voltage_code = XENON_EMMC_VCCQ_1_8V;
+
+ /*
+ * This host is for eMMC, XENON self-defined
+ * eMMC control register should be accessed
+ * instead of Host Control 2
+ */
+ ctrl = sdhci_readl(host, XENON_SLOT_EMMC_CTRL);
+ ctrl &= ~XENON_EMMC_VCCQ_MASK;
+ ctrl |= voltage_code;
+ sdhci_writel(host, ctrl, XENON_SLOT_EMMC_CTRL);
+
+ /* There is no standard to determine this waiting period */
+ usleep_range(1000, 2000);
+
+ /* Check whether io voltage switch is done */
+ ctrl = sdhci_readl(host, XENON_SLOT_EMMC_CTRL);
+ ctrl &= XENON_EMMC_VCCQ_MASK;
+ /*
+ * This bit is set only when regulator feeds back
+ * the voltage switch results to Xenon SDHC.
+ * However, in actaul implementation, regulator might not
+ * provide this feedback.
+ * Thus we shall not rely on this bit to determine
+ * if switch failed.
+ * If the bit is not set, just throw a message.
+ * Besides, error code should not be returned.
+ */
+ if (ctrl != voltage_code)
+ dev_info(mmc_dev(mmc), "fail to detect eMMC signal voltage stable\n");
+ return 0;
+ }
+
+ dev_err(mmc_dev(mmc), "Unsupported signal voltage: %d\n", voltage);
+ return -EINVAL;
+}
+
+static int xenon_start_signal_voltage_switch(struct mmc_host *mmc,
+ struct mmc_ios *ios)
+{
+ struct sdhci_host *host = mmc_priv(mmc);
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+
+ /*
+ * Before SD/SDIO set signal voltage, SD bus clock should be
+ * disabled. However, sdhci_set_clock will also disable the Internal
+ * clock in mmc_set_signal_voltage().
+ * If Internal clock is disabled, the 3.3V/1.8V bit can not be updated.
+ * Thus here manually enable internal clock.
+ *
+ * After switch completes, it is unnecessary to disable internal clock,
+ * since keeping internal clock active obeys SD spec.
+ */
+ enable_xenon_internal_clk(host);
+
+ if (priv->init_card_type == MMC_TYPE_MMC)
+ return xenon_emmc_signal_voltage_switch(mmc, ios);
+
+ return sdhci_start_signal_voltage_switch(mmc, ios);
+}
+
+/*
+ * Update card type.
+ * priv->init_card_type will be used in PHY timing adjustment.
+ */
+static void xenon_init_card(struct mmc_host *mmc, struct mmc_card *card)
+{
+ struct sdhci_host *host = mmc_priv(mmc);
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+
+ /* Update card type*/
+ priv->init_card_type = card->type;
+}
+
+static int xenon_execute_tuning(struct mmc_host *mmc, u32 opcode)
+{
+ struct sdhci_host *host = mmc_priv(mmc);
+
+ if (host->timing == MMC_TIMING_UHS_DDR50)
+ return 0;
+
+ /*
+ * Currently force Xenon driver back to support mode 1 only,
+ * even though Xenon might claim to support mode 2 or mode 3.
+ * It requires more time to test mode 2/mode 3 on more platforms.
+ */
+ if (host->tuning_mode != SDHCI_TUNING_MODE_1)
+ xenon_sdhc_retune_setup(host);
+
+ return sdhci_execute_tuning(mmc, opcode);
+}
+
+static void xenon_enable_sdio_irq(struct mmc_host *mmc, int enable)
+{
+ struct sdhci_host *host = mmc_priv(mmc);
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ u32 reg;
+ u8 sdhc_id = priv->sdhc_id;
+
+ sdhci_enable_sdio_irq(mmc, enable);
+
+ if (enable) {
+ /*
+ * Set SDIO Card Inserted indication
+ * to enable detecting SDIO async irq.
+ */
+ reg = sdhci_readl(host, XENON_SYS_CFG_INFO);
+ reg |= (1 << (sdhc_id + XENON_SLOT_TYPE_SDIO_SHIFT));
+ sdhci_writel(host, reg, XENON_SYS_CFG_INFO);
+ } else {
+ /* Clear SDIO Card Inserted indication */
+ reg = sdhci_readl(host, XENON_SYS_CFG_INFO);
+ reg &= ~(1 << (sdhc_id + XENON_SLOT_TYPE_SDIO_SHIFT));
+ sdhci_writel(host, reg, XENON_SYS_CFG_INFO);
+ }
+}
+
+static void xenon_replace_mmc_host_ops(struct sdhci_host *host)
+{
+ host->mmc_host_ops.set_ios = xenon_set_ios;
+ host->mmc_host_ops.start_signal_voltage_switch =
+ xenon_start_signal_voltage_switch;
+ host->mmc_host_ops.init_card = xenon_init_card;
+ host->mmc_host_ops.execute_tuning = xenon_execute_tuning;
+ host->mmc_host_ops.enable_sdio_irq = xenon_enable_sdio_irq;
+}
+
+/*
+ * Parse child node in Xenon DT.
+ * Search for the following item(s):
+ * - eMMC card type
+ */
+static int xenon_child_node_of_parse(struct platform_device *pdev)
+{
+ struct device_node *np = pdev->dev.of_node;
+ struct sdhci_host *host = platform_get_drvdata(pdev);
+ struct mmc_host *mmc = host->mmc;
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ struct device_node *child;
+ int nr_child;
+
+ priv->init_card_type = XENON_CARD_TYPE_UNKNOWN;
+
+ nr_child = of_get_child_count(np);
+ if (!nr_child)
+ return 0;
+
+ for_each_child_of_node(np, child) {
+ if (of_device_is_compatible(child, "mmc-card")) {
+ priv->init_card_type = MMC_TYPE_MMC;
+ mmc->caps |= MMC_CAP_NONREMOVABLE;
+
+ /*
+ * Force to clear BUS_TEST to
+ * skip bus_test_pre and bus_test_post
+ */
+ mmc->caps &= ~MMC_CAP_BUS_WIDTH_TEST;
+ mmc->caps2 |= MMC_CAP2_HC_ERASE_SZ |
+ MMC_CAP2_PACKED_CMD |
+ MMC_CAP2_NO_SD |
+ MMC_CAP2_NO_SDIO;
+ }
+ }
+
+ return 0;
+}
+
+static int xenon_probe_dt(struct platform_device *pdev)
+{
+ struct device_node *np = pdev->dev.of_node;
+ struct sdhci_host *host = platform_get_drvdata(pdev);
+ struct mmc_host *mmc = host->mmc;
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ int err;
+ u32 sdhc_id, nr_sdhc;
+ u32 tuning_count;
+
+ /* Standard MMC property */
+ err = mmc_of_parse(mmc);
+ if (err)
+ return err;
+
+ /* Standard SDHCI property */
+ sdhci_get_of_property(pdev);
+
+ /*
+ * Xenon Specific property:
+ * init_card_type: check whether this SDHC is for eMMC
+ * sdhc-id: the index of current SDHC.
+ * Refer to XENON_SYS_CFG_INFO register
+ * tun-count: the interval between re-tuning
+ */
+ /* Parse child node, including checking emmc type */
+ err = xenon_child_node_of_parse(pdev);
+ if (err)
+ return err;
+
+ priv->sdhc_id = 0x0;
+ if (!of_property_read_u32(np, "marvell,xenon-sdhc-id", &sdhc_id)) {
+ nr_sdhc = sdhci_readl(host, XENON_SYS_CFG_INFO);
+ nr_sdhc &= XENON_NR_SUPPORTED_SLOT_MASK;
+ if (unlikely(sdhc_id > nr_sdhc)) {
+ dev_err(mmc_dev(mmc), "SDHC Index %d exceeds Number of SDHCs %d\n",
+ sdhc_id, nr_sdhc);
+ return -EINVAL;
+ }
+ }
+
+ tuning_count = XENON_DEF_TUNING_COUNT;
+ if (!of_property_read_u32(np, "marvell,xenon-tun-count",
+ &tuning_count)) {
+ if (unlikely(tuning_count >= XENON_TMR_RETUN_NO_PRESENT)) {
+ dev_err(mmc_dev(mmc), "Wrong Re-tuning Count. Set default value %d\n",
+ XENON_DEF_TUNING_COUNT);
+ tuning_count = XENON_DEF_TUNING_COUNT;
+ }
+ }
+ priv->tuning_count = tuning_count;
+
+ return err;
+}
+
+static int xenon_sdhc_probe(struct sdhci_host *host)
+{
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ u8 sdhc_id = priv->sdhc_id;
+
+ /* Enable SDHC */
+ xenon_enable_sdhc(host, sdhc_id);
+
+ /* Enable ACG */
+ xenon_set_acg(host, true);
+
+ /* Enable Parallel Transfer Mode */
+ xenon_enable_sdhc_parallel_tran(host, sdhc_id);
+
+ xenon_mask_cmd_conflict_err(host);
+
+ return 0;
+}
+
+static void xenon_sdhc_remove(struct sdhci_host *host)
+{
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ u8 sdhc_id = priv->sdhc_id;
+
+ /* disable SDHC */
+ xenon_disable_sdhc(host, sdhc_id);
+}
+
+static int sdhci_xenon_probe(struct platform_device *pdev)
+{
+ struct sdhci_pltfm_host *pltfm_host;
+ struct sdhci_host *host;
+ struct sdhci_xenon_priv *priv;
+ int err;
+
+ host = sdhci_pltfm_init(pdev, &sdhci_xenon_pdata,
+ sizeof(struct sdhci_xenon_priv));
+ if (IS_ERR(host))
+ return PTR_ERR(host);
+
+ pltfm_host = sdhci_priv(host);
+ priv = sdhci_pltfm_priv(pltfm_host);
+
+ xenon_set_acg(host, false);
+
+ /*
+ * Link Xenon specific mmc_host_ops function,
+ * to replace standard ones in sdhci_ops.
+ */
+ xenon_replace_mmc_host_ops(host);
+
+ pltfm_host->clk = devm_clk_get(&pdev->dev, "core");
+ if (IS_ERR(pltfm_host->clk)) {
+ err = PTR_ERR(pltfm_host->clk);
+ dev_err(&pdev->dev, "Failed to setup input clk: %d\n", err);
+ goto free_pltfm;
+ }
+ err = clk_prepare_enable(pltfm_host->clk);
+ if (err)
+ goto free_pltfm;
+
+ err = xenon_probe_dt(pdev);
+ if (err)
+ goto err_clk;
+
+ err = xenon_sdhc_probe(host);
+ if (err)
+ goto err_clk;
+
+ err = sdhci_add_host(host);
+ if (err)
+ goto remove_sdhc;
+
+ return 0;
+
+remove_sdhc:
+ xenon_sdhc_remove(host);
+err_clk:
+ clk_disable_unprepare(pltfm_host->clk);
+free_pltfm:
+ sdhci_pltfm_free(pdev);
+ return err;
+}
+
+static int sdhci_xenon_remove(struct platform_device *pdev)
+{
+ struct sdhci_host *host = platform_get_drvdata(pdev);
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+
+ xenon_sdhc_remove(host);
+
+ sdhci_remove_host(host, 0);
+
+ clk_disable_unprepare(pltfm_host->clk);
+
+ sdhci_pltfm_free(pdev);
+
+ return 0;
+}
+
+static const struct of_device_id sdhci_xenon_dt_ids[] = {
+ { .compatible = "marvell,armada-8k-sdhci",},
+ { .compatible = "marvell,armada-3700-sdhci",},
+ {}
+};
+MODULE_DEVICE_TABLE(of, sdhci_xenon_dt_ids);
+
+static struct platform_driver sdhci_xenon_driver = {
+ .driver = {
+ .name = "xenon-sdhci",
+ .of_match_table = sdhci_xenon_dt_ids,
+ .pm = &sdhci_pltfm_pmops,
+ },
+ .probe = sdhci_xenon_probe,
+ .remove = sdhci_xenon_remove,
+};
+
+module_platform_driver(sdhci_xenon_driver);
+
+MODULE_DESCRIPTION("SDHCI platform driver for Marvell Xenon SDHC");
+MODULE_AUTHOR("Hu Ziji <huziji@marvell.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mmc/host/sdhci-xenon.h b/drivers/mmc/host/sdhci-xenon.h
new file mode 100644
index 000000000000..69de711db9eb
--- /dev/null
+++ b/drivers/mmc/host/sdhci-xenon.h
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2016 Marvell, All Rights Reserved.
+ *
+ * Author: Hu Ziji <huziji@marvell.com>
+ * Date: 2016-8-24
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ */
+#ifndef SDHCI_XENON_H_
+#define SDHCI_XENON_H_
+
+/* Register Offset of Xenon SDHC self-defined register */
+#define XENON_SYS_CFG_INFO 0x0104
+#define XENON_SLOT_TYPE_SDIO_SHIFT 24
+#define XENON_NR_SUPPORTED_SLOT_MASK 0x7
+
+#define XENON_SYS_OP_CTRL 0x0108
+#define XENON_AUTO_CLKGATE_DISABLE_MASK BIT(20)
+#define XENON_SDCLK_IDLEOFF_ENABLE_SHIFT 8
+#define XENON_SLOT_ENABLE_SHIFT 0
+
+#define XENON_SYS_EXT_OP_CTRL 0x010C
+#define XENON_MASK_CMD_CONFLICT_ERR BIT(8)
+
+#define XENON_SLOT_EMMC_CTRL 0x0130
+#define XENON_EMMC_VCCQ_MASK 0x3
+#define XENON_EMMC_VCCQ_1_8V 0x1
+#define XENON_EMMC_VCCQ_3_3V 0x3
+
+#define XENON_SLOT_RETUNING_REQ_CTRL 0x0144
+/* retuning compatible */
+#define XENON_RETUNING_COMPATIBLE 0x1
+
+/* Tuning Parameter */
+#define XENON_TMR_RETUN_NO_PRESENT 0xF
+#define XENON_DEF_TUNING_COUNT 0x9
+
+#define XENON_DEFAULT_SDCLK_FREQ 400000
+
+/* Xenon specific Mode Select value */
+#define XENON_CTRL_HS200 0x5
+#define XENON_CTRL_HS400 0x6
+
+/* Indicate Card Type is not clear yet */
+#define XENON_CARD_TYPE_UNKNOWN 0xF
+
+struct sdhci_xenon_priv {
+ unsigned char tuning_count;
+ /* idx of SDHC */
+ u8 sdhc_id;
+
+ /*
+ * eMMC/SD/SDIO require different PHY settings or
+ * voltage control. It's necessary for Xenon driver to
+ * recognize card type during, or even before initialization.
+ * However, mmc_host->card is not available yet at that time.
+ * This field records the card type during init.
+ * For eMMC, it is updated in dt parse. For SD/SDIO, it is
+ * updated in xenon_init_card().
+ *
+ * It is only valid during initialization after it is updated.
+ * Do not access this variable in normal transfers after
+ * initialization completes.
+ */
+ unsigned int init_card_type;
+};
+
+#endif
--
git-series 0.9.1
^ permalink raw reply related
* [PATCH v5 07/12] mmc: sdhci-xenon: Add support to PHYs of Marvell Xenon SDHC.
From: Gregory CLEMENT @ 2017-01-11 17:19 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <cover.8527229dae6c124f24c5e637430c2cdc86f80392.1484154449.git-series.gregory.clement@free-electrons.com>
From: Hu Ziji <huziji@marvell.com>
Marvell Xenon eMMC/SD/SDIO Host Controller contains PHY.
Multiple types of PHYs are supported.
Add support to multiple types of PHYs init and configuration.
Add register definitions of PHYs.
Xenon PHY cannot fit in kernel common PHY framework.
Xenon SDHC PHY register is a part of Xenon SDHC register set.
Besides, MMC initialization has to call several PHY functions to
complete timing setting.
Those PHY setting functions have to access SDHC registers and know
current MMC setting, such as bus width, clock frequency and
speed mode.
As a result, implement Xenon PHY in MMC host directory.
Signed-off-by: Hu Ziji <huziji@marvell.com>
Tested-by: Russell King <rmk+kernel@armlinux.org.uk>
Signed-off-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
---
drivers/mmc/host/Makefile | 2 +-
drivers/mmc/host/sdhci-xenon-phy.c | 790 ++++++++++++++++++++++++++++++-
drivers/mmc/host/sdhci-xenon.c | 3 +-
drivers/mmc/host/sdhci-xenon.h | 37 +-
4 files changed, 830 insertions(+), 2 deletions(-)
create mode 100644 drivers/mmc/host/sdhci-xenon-phy.c
diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile
index b0a2ab4b256e..893b48db5513 100644
--- a/drivers/mmc/host/Makefile
+++ b/drivers/mmc/host/Makefile
@@ -84,4 +84,4 @@ ifeq ($(CONFIG_CB710_DEBUG),y)
endif
obj-$(CONFIG_MMC_SDHCI_XENON) += sdhci-xenon-driver.o
-sdhci-xenon-driver-y += sdhci-xenon.o
+sdhci-xenon-driver-y += sdhci-xenon.o sdhci-xenon-phy.o
diff --git a/drivers/mmc/host/sdhci-xenon-phy.c b/drivers/mmc/host/sdhci-xenon-phy.c
new file mode 100644
index 000000000000..f410712d24ea
--- /dev/null
+++ b/drivers/mmc/host/sdhci-xenon-phy.c
@@ -0,0 +1,790 @@
+/*
+ * PHY support for Xenon SDHC
+ *
+ * Copyright (C) 2016 Marvell, All Rights Reserved.
+ *
+ * Author: Hu Ziji <huziji@marvell.com>
+ * Date: 2016-8-24
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ */
+
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/of_address.h>
+
+#include "sdhci-pltfm.h"
+#include "sdhci-xenon.h"
+
+/* Register base for eMMC PHY 5.0 Version */
+#define XENON_EMMC_5_0_PHY_REG_BASE 0x0160
+/* Register base for eMMC PHY 5.1 Version */
+#define XENON_EMMC_PHY_REG_BASE 0x0170
+
+#define XENON_EMMC_PHY_TIMING_ADJUST XENON_EMMC_PHY_REG_BASE
+#define XENON_EMMC_5_0_PHY_TIMING_ADJUST XENON_EMMC_5_0_PHY_REG_BASE
+#define XENON_TIMING_ADJUST_SLOW_MODE BIT(29)
+#define XENON_TIMING_ADJUST_SDIO_MODE BIT(28)
+#define XENON_OUTPUT_QSN_PHASE_SELECT BIT(17)
+#define XENON_SAMPL_INV_QSP_PHASE_SELECT BIT(18)
+#define XENON_SAMPL_INV_QSP_PHASE_SELECT_SHIFT 18
+#define XENON_PHY_INITIALIZAION BIT(31)
+#define XENON_WAIT_CYCLE_BEFORE_USING_MASK 0xF
+#define XENON_WAIT_CYCLE_BEFORE_USING_SHIFT 12
+#define XENON_FC_SYNC_EN_DURATION_MASK 0xF
+#define XENON_FC_SYNC_EN_DURATION_SHIFT 8
+#define XENON_FC_SYNC_RST_EN_DURATION_MASK 0xF
+#define XENON_FC_SYNC_RST_EN_DURATION_SHIFT 4
+#define XENON_FC_SYNC_RST_DURATION_MASK 0xF
+#define XENON_FC_SYNC_RST_DURATION_SHIFT 0
+
+#define XENON_EMMC_PHY_FUNC_CONTROL (XENON_EMMC_PHY_REG_BASE + 0x4)
+#define XENON_EMMC_5_0_PHY_FUNC_CONTROL \
+ (XENON_EMMC_5_0_PHY_REG_BASE + 0x4)
+#define XENON_ASYNC_DDRMODE_MASK BIT(23)
+#define XENON_ASYNC_DDRMODE_SHIFT 23
+#define XENON_CMD_DDR_MODE BIT(16)
+#define XENON_DQ_DDR_MODE_SHIFT 8
+#define XENON_DQ_DDR_MODE_MASK 0xFF
+#define XENON_DQ_ASYNC_MODE BIT(4)
+
+#define XENON_EMMC_PHY_PAD_CONTROL (XENON_EMMC_PHY_REG_BASE + 0x8)
+#define XENON_EMMC_5_0_PHY_PAD_CONTROL \
+ (XENON_EMMC_5_0_PHY_REG_BASE + 0x8)
+#define XENON_REC_EN_SHIFT 24
+#define XENON_REC_EN_MASK 0xF
+#define XENON_FC_DQ_RECEN BIT(24)
+#define XENON_FC_CMD_RECEN BIT(25)
+#define XENON_FC_QSP_RECEN BIT(26)
+#define XENON_FC_QSN_RECEN BIT(27)
+#define XENON_OEN_QSN BIT(28)
+#define XENON_AUTO_RECEN_CTRL BIT(30)
+#define XENON_FC_ALL_CMOS_RECEIVER 0xF000
+
+#define XENON_EMMC5_FC_QSP_PD BIT(18)
+#define XENON_EMMC5_FC_QSP_PU BIT(22)
+#define XENON_EMMC5_FC_CMD_PD BIT(17)
+#define XENON_EMMC5_FC_CMD_PU BIT(21)
+#define XENON_EMMC5_FC_DQ_PD BIT(16)
+#define XENON_EMMC5_FC_DQ_PU BIT(20)
+
+#define XENON_EMMC_PHY_PAD_CONTROL1 (XENON_EMMC_PHY_REG_BASE + 0xC)
+#define XENON_EMMC5_1_FC_QSP_PD BIT(9)
+#define XENON_EMMC5_1_FC_QSP_PU BIT(25)
+#define XENON_EMMC5_1_FC_CMD_PD BIT(8)
+#define XENON_EMMC5_1_FC_CMD_PU BIT(24)
+#define XENON_EMMC5_1_FC_DQ_PD 0xFF
+#define XENON_EMMC5_1_FC_DQ_PU (0xFF << 16)
+
+#define XENON_EMMC_PHY_PAD_CONTROL2 (XENON_EMMC_PHY_REG_BASE + 0x10)
+#define XENON_EMMC_5_0_PHY_PAD_CONTROL2 \
+ (XENON_EMMC_5_0_PHY_REG_BASE + 0xC)
+#define XENON_ZNR_MASK 0x1F
+#define XENON_ZNR_SHIFT 8
+#define XENON_ZPR_MASK 0x1F
+/* Perferred ZNR and ZPR value vary between different boards.
+ * The specific ZNR and ZPR value should be defined here
+ * according to board actual timing.
+ */
+#define XENON_ZNR_DEF_VALUE 0xF
+#define XENON_ZPR_DEF_VALUE 0xF
+
+#define XENON_EMMC_PHY_DLL_CONTROL (XENON_EMMC_PHY_REG_BASE + 0x14)
+#define XENON_EMMC_5_0_PHY_DLL_CONTROL \
+ (XENON_EMMC_5_0_PHY_REG_BASE + 0x10)
+#define XENON_DLL_ENABLE BIT(31)
+#define XENON_DLL_UPDATE_STROBE_5_0 BIT(30)
+#define XENON_DLL_REFCLK_SEL BIT(30)
+#define XENON_DLL_UPDATE BIT(23)
+#define XENON_DLL_PHSEL1_SHIFT 24
+#define XENON_DLL_PHSEL0_SHIFT 16
+#define XENON_DLL_PHASE_MASK 0x3F
+#define XENON_DLL_PHASE_90_DEGREE 0x1F
+#define XENON_DLL_FAST_LOCK BIT(5)
+#define XENON_DLL_GAIN2X BIT(3)
+#define XENON_DLL_BYPASS_EN BIT(0)
+
+#define XENON_EMMC_5_0_PHY_LOGIC_TIMING_ADJUST \
+ (XENON_EMMC_5_0_PHY_REG_BASE + 0x14)
+#define XENON_EMMC_PHY_LOGIC_TIMING_ADJUST (XENON_EMMC_PHY_REG_BASE + 0x18)
+#define XENON_LOGIC_TIMING_VALUE 0x00AA8977
+
+/*
+ * List offset of PHY registers and some special register values
+ * in eMMC PHY 5.0 or eMMC PHY 5.1
+ */
+struct xenon_emmc_phy_regs {
+ /* Offset of Timing Adjust register */
+ u16 timing_adj;
+ /* Offset of Func Control register */
+ u16 func_ctrl;
+ /* Offset of Pad Control register */
+ u16 pad_ctrl;
+ /* Offset of Pad Control register 2 */
+ u16 pad_ctrl2;
+ /* Offset of DLL Control register */
+ u16 dll_ctrl;
+ /* Offset of Logic Timing Adjust register */
+ u16 logic_timing_adj;
+ /* DLL Update Enable bit */
+ u32 dll_update;
+};
+
+static const char * const phy_types[] = {
+ "emmc 5.0 phy",
+ "emmc 5.1 phy"
+};
+
+enum phy_type_enum {
+ EMMC_5_0_PHY,
+ EMMC_5_1_PHY,
+ NR_PHY_TYPES
+};
+
+static struct xenon_emmc_phy_regs xenon_emmc_5_0_phy_regs = {
+ .timing_adj = XENON_EMMC_5_0_PHY_TIMING_ADJUST,
+ .func_ctrl = XENON_EMMC_5_0_PHY_FUNC_CONTROL,
+ .pad_ctrl = XENON_EMMC_5_0_PHY_PAD_CONTROL,
+ .pad_ctrl2 = XENON_EMMC_5_0_PHY_PAD_CONTROL2,
+ .dll_ctrl = XENON_EMMC_5_0_PHY_DLL_CONTROL,
+ .logic_timing_adj = XENON_EMMC_5_0_PHY_LOGIC_TIMING_ADJUST,
+ .dll_update = XENON_DLL_UPDATE_STROBE_5_0,
+};
+
+static struct xenon_emmc_phy_regs xenon_emmc_5_1_phy_regs = {
+ .timing_adj = XENON_EMMC_PHY_TIMING_ADJUST,
+ .func_ctrl = XENON_EMMC_PHY_FUNC_CONTROL,
+ .pad_ctrl = XENON_EMMC_PHY_PAD_CONTROL,
+ .pad_ctrl2 = XENON_EMMC_PHY_PAD_CONTROL2,
+ .dll_ctrl = XENON_EMMC_PHY_DLL_CONTROL,
+ .logic_timing_adj = XENON_EMMC_PHY_LOGIC_TIMING_ADJUST,
+ .dll_update = XENON_DLL_UPDATE,
+};
+
+/*
+ * eMMC PHY configuration and operations
+ */
+struct emmc_phy_params {
+ bool slow_mode;
+
+ u8 znr;
+ u8 zpr;
+
+ /* Nr of consecutive Sampling Points of a Valid Sampling Window */
+ u8 nr_tun_times;
+ /* Divider for calculating Tuning Step */
+ u8 tun_step_divider;
+};
+
+static int alloc_emmc_phy(struct sdhci_xenon_priv *priv)
+{
+ struct emmc_phy_params *params;
+
+ params = kzalloc(sizeof(*params), GFP_KERNEL);
+ if (!params)
+ return -ENOMEM;
+
+ priv->phy_params = params;
+ if (priv->phy_type == EMMC_5_0_PHY)
+ priv->emmc_phy_regs = &xenon_emmc_5_0_phy_regs;
+ else
+ priv->emmc_phy_regs = &xenon_emmc_5_1_phy_regs;
+
+ return 0;
+}
+
+/*
+ * eMMC 5.0/5.1 PHY init/re-init.
+ * eMMC PHY init should be executed after:
+ * 1. SDCLK frequecny changes.
+ * 2. SDCLK is stopped and re-enabled.
+ * 3. config in emmc_phy_regs->timing_adj and emmc_phy_regs->func_ctrl
+ * are changed
+ */
+static int emmc_phy_init(struct sdhci_host *host)
+{
+ u32 reg;
+ u32 wait, clock;
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ struct xenon_emmc_phy_regs *phy_regs = priv->emmc_phy_regs;
+
+ reg = sdhci_readl(host, phy_regs->timing_adj);
+ reg |= XENON_PHY_INITIALIZAION;
+ sdhci_writel(host, reg, phy_regs->timing_adj);
+
+ /* Add duration of FC_SYNC_RST */
+ wait = ((reg >> XENON_FC_SYNC_RST_DURATION_SHIFT) &
+ XENON_FC_SYNC_RST_DURATION_MASK);
+ /* Add interval between FC_SYNC_EN and FC_SYNC_RST */
+ wait += ((reg >> XENON_FC_SYNC_RST_EN_DURATION_SHIFT) &
+ XENON_FC_SYNC_RST_EN_DURATION_MASK);
+ /* Add duration of asserting FC_SYNC_EN */
+ wait += ((reg >> XENON_FC_SYNC_EN_DURATION_SHIFT) &
+ XENON_FC_SYNC_EN_DURATION_MASK);
+ /* Add duration of waiting for PHY */
+ wait += ((reg >> XENON_WAIT_CYCLE_BEFORE_USING_SHIFT) &
+ XENON_WAIT_CYCLE_BEFORE_USING_MASK);
+ /* 4 addtional bus clock and 4 AXI bus clock are required */
+ wait += 8;
+ wait <<= 20;
+
+ clock = host->clock;
+ if (!clock)
+ /* Use the possibly slowest bus frequency value */
+ clock = XENON_LOWEST_SDCLK_FREQ;
+ /* get the wait time */
+ wait /= clock;
+ wait++;
+ /* wait for host eMMC PHY init completes */
+ udelay(wait);
+
+ reg = sdhci_readl(host, phy_regs->timing_adj);
+ reg &= XENON_PHY_INITIALIZAION;
+ if (reg) {
+ dev_err(mmc_dev(host->mmc), "eMMC PHY init cannot complete after %d us\n",
+ wait);
+ return -ETIMEDOUT;
+ }
+
+ return 0;
+}
+
+/*
+ * Enable eMMC PHY HW DLL
+ * DLL should be enabled and stable before HS200/SDR104 tuning,
+ * and before HS400 data strobe setting.
+ */
+static int emmc_phy_enable_dll(struct sdhci_host *host)
+{
+ u32 reg;
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ struct xenon_emmc_phy_regs *phy_regs = priv->emmc_phy_regs;
+ u8 timeout;
+
+ if (WARN_ON(host->clock <= MMC_HIGH_52_MAX_DTR))
+ return -EINVAL;
+
+ reg = sdhci_readl(host, phy_regs->dll_ctrl);
+ if (reg & XENON_DLL_ENABLE)
+ return 0;
+
+ /* Enable DLL */
+ reg = sdhci_readl(host, phy_regs->dll_ctrl);
+ reg |= (XENON_DLL_ENABLE | XENON_DLL_FAST_LOCK);
+
+ /*
+ * Set Phase as 90 degree, which is most common value.
+ * Might set another value if necessary.
+ * The granularity is 1 degree.
+ */
+ reg &= ~((XENON_DLL_PHASE_MASK << XENON_DLL_PHSEL0_SHIFT) |
+ (XENON_DLL_PHASE_MASK << XENON_DLL_PHSEL1_SHIFT));
+ reg |= ((XENON_DLL_PHASE_90_DEGREE << XENON_DLL_PHSEL0_SHIFT) |
+ (XENON_DLL_PHASE_90_DEGREE << XENON_DLL_PHSEL1_SHIFT));
+
+ reg &= ~XENON_DLL_BYPASS_EN;
+ reg |= phy_regs->dll_update;
+ if (priv->phy_type == EMMC_5_1_PHY)
+ reg &= ~XENON_DLL_REFCLK_SEL;
+ sdhci_writel(host, reg, phy_regs->dll_ctrl);
+
+ /* Wait max 32 ms */
+ timeout = 32;
+ while (!(sdhci_readw(host, XENON_SLOT_EXT_PRESENT_STATE) &
+ XENON_DLL_LOCK_STATE)) {
+ if (!timeout) {
+ dev_err(mmc_dev(host->mmc), "Wait for DLL Lock time-out\n");
+ return -ETIMEDOUT;
+ }
+ timeout--;
+ mdelay(1);
+ }
+ return 0;
+}
+
+/*
+ * Config to eMMC PHY to prepare for tuning.
+ * Enable HW DLL and set the TUNING_STEP
+ */
+static int emmc_phy_config_tuning(struct sdhci_host *host)
+{
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ struct emmc_phy_params *params = priv->phy_params;
+ u32 reg, tuning_step;
+ int ret;
+ unsigned long flags;
+
+ if (WARN_ON(host->clock <= MMC_HIGH_52_MAX_DTR))
+ return -EINVAL;
+
+ spin_lock_irqsave(&host->lock, flags);
+
+ ret = emmc_phy_enable_dll(host);
+ if (ret) {
+ spin_unlock_irqrestore(&host->lock, flags);
+ return ret;
+ }
+
+ /* Achieve TUNGING_STEP with HW DLL help */
+ reg = sdhci_readl(host, XENON_SLOT_DLL_CUR_DLY_VAL);
+ tuning_step = reg / params->tun_step_divider;
+ if (unlikely(tuning_step > XENON_TUNING_STEP_MASK)) {
+ dev_warn(mmc_dev(host->mmc),
+ "HS200 TUNING_STEP %d is larger than MAX value\n",
+ tuning_step);
+ tuning_step = XENON_TUNING_STEP_MASK;
+ }
+
+ /* Set TUNING_STEP for later tuning */
+ reg = sdhci_readl(host, XENON_SLOT_OP_STATUS_CTRL);
+ reg &= ~(XENON_TUN_CONSECUTIVE_TIMES_MASK <<
+ XENON_TUN_CONSECUTIVE_TIMES_SHIFT);
+ reg |= (params->nr_tun_times << XENON_TUN_CONSECUTIVE_TIMES_SHIFT);
+ reg &= ~(XENON_TUNING_STEP_MASK << XENON_TUNING_STEP_SHIFT);
+ reg |= (tuning_step << XENON_TUNING_STEP_SHIFT);
+ sdhci_writel(host, reg, XENON_SLOT_OP_STATUS_CTRL);
+
+ spin_unlock_irqrestore(&host->lock, flags);
+ return 0;
+}
+
+static void __emmc_phy_disable_data_strobe(struct sdhci_host *host)
+{
+ u32 reg;
+
+ /* Disable SDHC Data Strobe */
+ reg = sdhci_readl(host, XENON_SLOT_EMMC_CTRL);
+ reg &= ~XENON_ENABLE_DATA_STROBE;
+ sdhci_writel(host, reg, XENON_SLOT_EMMC_CTRL);
+}
+
+/* Set HS400 Data Strobe */
+static void emmc_phy_strobe_delay_adj(struct sdhci_host *host)
+{
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ unsigned long flags;
+ u32 reg;
+
+ if (WARN_ON(host->timing != MMC_TIMING_MMC_HS400))
+ return;
+
+ if (host->clock <= MMC_HIGH_52_MAX_DTR)
+ return;
+
+ dev_dbg(mmc_dev(host->mmc), "starts HS400 strobe delay adjustment\n");
+
+ spin_lock_irqsave(&host->lock, flags);
+
+ emmc_phy_enable_dll(host);
+
+ /* Enable SDHC Data Strobe */
+ reg = sdhci_readl(host, XENON_SLOT_EMMC_CTRL);
+ reg |= XENON_ENABLE_DATA_STROBE;
+ sdhci_writel(host, reg, XENON_SLOT_EMMC_CTRL);
+
+ /* Set Data Strobe Pull down */
+ if (priv->phy_type == EMMC_5_0_PHY) {
+ reg = sdhci_readl(host, XENON_EMMC_5_0_PHY_PAD_CONTROL);
+ reg |= XENON_EMMC5_FC_QSP_PD;
+ reg &= ~XENON_EMMC5_FC_QSP_PU;
+ sdhci_writel(host, reg, XENON_EMMC_5_0_PHY_PAD_CONTROL);
+ } else {
+ reg = sdhci_readl(host, XENON_EMMC_PHY_PAD_CONTROL1);
+ reg |= XENON_EMMC5_1_FC_QSP_PD;
+ reg &= ~XENON_EMMC5_1_FC_QSP_PU;
+ sdhci_writel(host, reg, XENON_EMMC_PHY_PAD_CONTROL1);
+ }
+ spin_unlock_irqrestore(&host->lock, flags);
+}
+
+static inline bool temp_stage_hs200_to_hs400(struct sdhci_host *host,
+ struct sdhci_xenon_priv *priv)
+{
+ /*
+ * Tmep stages from HS200 to HS400
+ * from HS200 to HS in 200MHz
+ * from 200MHz to 52MHz
+ */
+ if (((priv->timing == MMC_TIMING_MMC_HS200) &&
+ (host->timing == MMC_TIMING_MMC_HS)) ||
+ ((host->timing == MMC_TIMING_MMC_HS) &&
+ (priv->clock > host->clock)))
+ return true;
+
+ return false;
+}
+
+static inline bool temp_stage_hs400_to_h200(struct sdhci_host *host,
+ struct sdhci_xenon_priv *priv)
+{
+ /*
+ * Temp stages from HS400 t0 HS200:
+ * from 200MHz to 52MHz in HS400
+ * from HS400 to HS DDR in 52MHz
+ * from HS DDR to HS in 52MHz
+ * from HS to HS200 in 52MHz
+ */
+ if (((priv->timing == MMC_TIMING_MMC_HS400) &&
+ ((host->clock == MMC_HIGH_52_MAX_DTR) ||
+ (host->timing == MMC_TIMING_MMC_DDR52))) ||
+ ((priv->timing == MMC_TIMING_MMC_DDR52) &&
+ (host->timing == MMC_TIMING_MMC_HS)) ||
+ ((host->timing == MMC_TIMING_MMC_HS200) &&
+ (host->clock == MMC_HIGH_52_MAX_DTR)))
+ return true;
+
+ return false;
+}
+
+/*
+ * If eMMC PHY Slow Mode is required in lower speed mode in SDR mode
+ * (SDLCK < 55MHz), enable Slow Mode to bypass eMMC PHY.
+ * SDIO slower SDR mode also requires Slow Mode.
+ *
+ * If Slow Mode is enabled, return true.
+ * Otherwise, return false.
+ */
+static bool emmc_phy_slow_mode(struct sdhci_host *host,
+ unsigned char timing)
+{
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ struct emmc_phy_params *params = priv->phy_params;
+ struct xenon_emmc_phy_regs *phy_regs = priv->emmc_phy_regs;
+ u32 reg;
+
+ /* Skip temp stages from HS200 to HS400 */
+ if (temp_stage_hs200_to_hs400(host, priv))
+ return false;
+
+ /* Skip temp stages from HS400 t0 HS200 */
+ if (temp_stage_hs400_to_h200(host, priv))
+ return false;
+
+ reg = sdhci_readl(host, phy_regs->timing_adj);
+ /* Enable Slow Mode for SDIO in slower SDR mode */
+ if ((priv->init_card_type == MMC_TYPE_SDIO) &&
+ ((timing == MMC_TIMING_UHS_SDR25) ||
+ (timing == MMC_TIMING_UHS_SDR12) ||
+ (timing == MMC_TIMING_SD_HS) ||
+ (timing == MMC_TIMING_LEGACY))) {
+ reg |= XENON_TIMING_ADJUST_SLOW_MODE;
+ sdhci_writel(host, reg, phy_regs->timing_adj);
+ return true;
+ }
+
+ /* Check if Slow Mode is required in lower speed mode in SDR mode */
+ if (((timing == MMC_TIMING_UHS_SDR50) ||
+ (timing == MMC_TIMING_UHS_SDR25) ||
+ (timing == MMC_TIMING_UHS_SDR12) ||
+ (timing == MMC_TIMING_SD_HS) ||
+ (timing == MMC_TIMING_MMC_HS) ||
+ (timing == MMC_TIMING_LEGACY)) && params->slow_mode) {
+ reg |= XENON_TIMING_ADJUST_SLOW_MODE;
+ sdhci_writel(host, reg, phy_regs->timing_adj);
+ return true;
+ }
+
+ reg &= ~XENON_TIMING_ADJUST_SLOW_MODE;
+ sdhci_writel(host, reg, phy_regs->timing_adj);
+ return false;
+}
+
+/*
+ * Set-up eMMC 5.0/5.1 PHY.
+ * Specific onfiguration depends on the current speed mode in use.
+ */
+static void emmc_phy_set(struct sdhci_host *host,
+ unsigned char timing)
+{
+ u32 reg;
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ struct emmc_phy_params *params = priv->phy_params;
+ struct xenon_emmc_phy_regs *phy_regs = priv->emmc_phy_regs;
+ unsigned long flags;
+
+ dev_dbg(mmc_dev(host->mmc), "eMMC PHY setting starts\n");
+
+ spin_lock_irqsave(&host->lock, flags);
+
+ /* Setup pad, set bit[28] and bits[26:24] */
+ reg = sdhci_readl(host, phy_regs->pad_ctrl);
+ reg |= (XENON_FC_DQ_RECEN | XENON_FC_CMD_RECEN |
+ XENON_FC_QSP_RECEN | XENON_OEN_QSN);
+ /* All FC_XX_RECEIVCE should be set as CMOS Type */
+ reg |= XENON_FC_ALL_CMOS_RECEIVER;
+ sdhci_writel(host, reg, phy_regs->pad_ctrl);
+
+ /* Set CMD and DQ Pull Up */
+ if (priv->phy_type == EMMC_5_0_PHY) {
+ reg = sdhci_readl(host, XENON_EMMC_5_0_PHY_PAD_CONTROL);
+ reg |= (XENON_EMMC5_FC_CMD_PU | XENON_EMMC5_FC_DQ_PU);
+ reg &= ~(XENON_EMMC5_FC_CMD_PD | XENON_EMMC5_FC_DQ_PD);
+ sdhci_writel(host, reg, XENON_EMMC_5_0_PHY_PAD_CONTROL);
+ } else {
+ reg = sdhci_readl(host, XENON_EMMC_PHY_PAD_CONTROL1);
+ reg |= (XENON_EMMC5_1_FC_CMD_PU | XENON_EMMC5_1_FC_DQ_PU);
+ reg &= ~(XENON_EMMC5_1_FC_CMD_PD | XENON_EMMC5_1_FC_DQ_PD);
+ sdhci_writel(host, reg, XENON_EMMC_PHY_PAD_CONTROL1);
+ }
+
+ if (timing == MMC_TIMING_LEGACY)
+ goto phy_init;
+
+ /*
+ * FIXME: should depends on the specific board timing.
+ */
+ if ((timing == MMC_TIMING_MMC_HS400) ||
+ (timing == MMC_TIMING_MMC_HS200) ||
+ (timing == MMC_TIMING_UHS_SDR50) ||
+ (timing == MMC_TIMING_UHS_SDR104) ||
+ (timing == MMC_TIMING_UHS_DDR50) ||
+ (timing == MMC_TIMING_UHS_SDR25) ||
+ (timing == MMC_TIMING_MMC_DDR52)) {
+ reg = sdhci_readl(host, phy_regs->timing_adj);
+ reg &= ~XENON_OUTPUT_QSN_PHASE_SELECT;
+ sdhci_writel(host, reg, phy_regs->timing_adj);
+ }
+
+ /*
+ * If SDIO card, set SDIO Mode
+ * Otherwise, clear SDIO Mode
+ */
+ reg = sdhci_readl(host, phy_regs->timing_adj);
+ if (priv->init_card_type == MMC_TYPE_SDIO)
+ reg |= XENON_TIMING_ADJUST_SDIO_MODE;
+ else
+ reg &= ~XENON_TIMING_ADJUST_SDIO_MODE;
+ sdhci_writel(host, reg, phy_regs->timing_adj);
+
+ if (emmc_phy_slow_mode(host, timing))
+ goto phy_init;
+
+ /*
+ * Set preferred ZNR and ZPR value
+ * The ZNR and ZPR value vary between different boards.
+ * Define them both in sdhci-xenon-emmc-phy.h.
+ */
+ reg = sdhci_readl(host, phy_regs->pad_ctrl2);
+ reg &= ~((XENON_ZNR_MASK << XENON_ZNR_SHIFT) | XENON_ZPR_MASK);
+ reg |= ((params->znr << XENON_ZNR_SHIFT) | params->zpr);
+ sdhci_writel(host, reg, phy_regs->pad_ctrl2);
+
+ /*
+ * When setting EMMC_PHY_FUNC_CONTROL register,
+ * SD clock should be disabled
+ */
+ reg = sdhci_readl(host, SDHCI_CLOCK_CONTROL);
+ reg &= ~SDHCI_CLOCK_CARD_EN;
+ sdhci_writew(host, reg, SDHCI_CLOCK_CONTROL);
+
+ reg = sdhci_readl(host, phy_regs->func_ctrl);
+ if ((timing == MMC_TIMING_UHS_DDR50) ||
+ (timing == MMC_TIMING_MMC_HS400) ||
+ (timing == MMC_TIMING_MMC_DDR52))
+ reg |= (XENON_DQ_DDR_MODE_MASK << XENON_DQ_DDR_MODE_SHIFT) |
+ XENON_CMD_DDR_MODE;
+ else
+ reg &= ~((XENON_DQ_DDR_MODE_MASK << XENON_DQ_DDR_MODE_SHIFT) |
+ XENON_CMD_DDR_MODE);
+
+ if (timing == MMC_TIMING_MMC_HS400)
+ reg &= ~XENON_DQ_ASYNC_MODE;
+ else
+ reg |= XENON_DQ_ASYNC_MODE;
+ sdhci_writel(host, reg, phy_regs->func_ctrl);
+
+ /* Enable bus clock */
+ reg = sdhci_readl(host, SDHCI_CLOCK_CONTROL);
+ reg |= SDHCI_CLOCK_CARD_EN;
+ sdhci_writew(host, reg, SDHCI_CLOCK_CONTROL);
+
+ if (timing == MMC_TIMING_MMC_HS400)
+ /* Hardware team recommend a value for HS400 */
+ sdhci_writel(host, XENON_LOGIC_TIMING_VALUE,
+ phy_regs->logic_timing_adj);
+ else
+ __emmc_phy_disable_data_strobe(host);
+
+phy_init:
+ emmc_phy_init(host);
+
+ spin_unlock_irqrestore(&host->lock, flags);
+
+ dev_dbg(mmc_dev(host->mmc), "eMMC PHY setting completes\n");
+}
+
+static int emmc_phy_parse_param_dt(struct sdhci_host *host,
+ struct device_node *np,
+ struct emmc_phy_params *params)
+{
+ u32 value;
+
+ if (of_property_read_bool(np, "marvell,xenon-phy-slow-mode"))
+ params->slow_mode = true;
+ else
+ params->slow_mode = false;
+
+ if (!of_property_read_u32(np, "marvell,xenon-phy-znr", &value))
+ params->znr = value & XENON_ZNR_MASK;
+ else
+ params->znr = XENON_ZNR_DEF_VALUE;
+
+ if (!of_property_read_u32(np, "marvell,xenon-phy-zpr", &value))
+ params->zpr = value & XENON_ZPR_MASK;
+ else
+ params->zpr = XENON_ZPR_DEF_VALUE;
+
+ if (!of_property_read_u32(np, "marvell,xenon-phy-nr-success-tun",
+ &value))
+ params->nr_tun_times = value & XENON_TUN_CONSECUTIVE_TIMES_MASK;
+ else
+ params->nr_tun_times = XENON_TUN_CONSECUTIVE_TIMES;
+
+ if (!of_property_read_u32(np, "marvell,xenon-phy-tun-step-divider",
+ &value))
+ params->tun_step_divider = value & 0xFF;
+ else
+ params->tun_step_divider = XENON_TUNING_STEP_DIVIDER;
+
+ return 0;
+}
+
+/*
+ * Setting PHY when card is working in High Speed Mode.
+ * HS400 set data strobe line.
+ * HS200/SDR104 set tuning config to prepare for tuning.
+ */
+static int xenon_hs_delay_adj(struct sdhci_host *host)
+{
+ int ret = 0;
+
+ if (WARN_ON(host->clock <= XENON_DEFAULT_SDCLK_FREQ))
+ return -EINVAL;
+
+ if (host->timing == MMC_TIMING_MMC_HS400) {
+ emmc_phy_strobe_delay_adj(host);
+ return 0;
+ }
+
+ if ((host->timing == MMC_TIMING_MMC_HS200) ||
+ (host->timing == MMC_TIMING_UHS_SDR104)) {
+ ret = emmc_phy_config_tuning(host);
+ if (!ret)
+ return 0;
+ }
+
+ /*
+ * DDR Mode requires driver to scan Sampling Fixed Delay Line,
+ * to find out a perfect operation sampling point.
+ * It is hard to implement such a scan in host driver since initiating
+ * commands by host driver is not safe.
+ * Thus so far just keep PHY Sampling Fixed Delay in default value
+ * in DDR mode.
+ *
+ * If any timing issue occrus in DDR mode on Marvell products,
+ * please contact maintainer to ask for internal support in Marvell.
+ */
+ if ((host->timing == MMC_TIMING_MMC_DDR52) ||
+ (host->timing == MMC_TIMING_UHS_DDR50))
+ dev_warn(mmc_dev(host->mmc), "Timing issue might occur in DDR mode\n");
+ return ret;
+}
+
+/*
+ * Adjust PHY setting.
+ * PHY setting should be adjusted when SDCLK frequency, Bus Width
+ * or Speed Mode is changed.
+ * Addtional config are required when card is working in High Speed mode,
+ * after leaving Legacy Mode.
+ */
+int xenon_phy_adj(struct sdhci_host *host, struct mmc_ios *ios)
+{
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ int ret = 0;
+
+ if (!host->clock) {
+ priv->clock = 0;
+ return 0;
+ }
+
+ /*
+ * The timing, frequency or bus width is changed,
+ * better to set eMMC PHY based on current setting
+ * and adjust Xenon SDHC delay.
+ */
+ if ((host->clock == priv->clock) &&
+ (ios->bus_width == priv->bus_width) &&
+ (ios->timing == priv->timing))
+ return 0;
+
+ emmc_phy_set(host, ios->timing);
+
+ /* Update the record */
+ priv->bus_width = ios->bus_width;
+
+ /* Skip temp stages from HS200 to HS400 */
+ if (temp_stage_hs200_to_hs400(host, priv))
+ return 0;
+
+ /* Skip temp stages from HS400 t0 HS200 */
+ if (temp_stage_hs400_to_h200(host, priv))
+ return 0;
+
+ priv->timing = ios->timing;
+ priv->clock = host->clock;
+
+ /* Legacy mode is a special case */
+ if (ios->timing == MMC_TIMING_LEGACY)
+ return 0;
+
+ if (host->clock > XENON_DEFAULT_SDCLK_FREQ)
+ ret = xenon_hs_delay_adj(host);
+ return ret;
+}
+
+static int add_xenon_phy(struct device_node *np, struct sdhci_host *host,
+ const char *phy_name)
+{
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ int i, ret;
+
+ for (i = 0; i < NR_PHY_TYPES; i++) {
+ if (!strcmp(phy_name, phy_types[i])) {
+ priv->phy_type = i;
+ break;
+ }
+ }
+ if (i == NR_PHY_TYPES) {
+ dev_err(mmc_dev(host->mmc),
+ "Unable to determine PHY name %s. Use default eMMC 5.1 PHY\n",
+ phy_name);
+ priv->phy_type = EMMC_5_1_PHY;
+ }
+
+ ret = alloc_emmc_phy(priv);
+ if (ret)
+ return ret;
+
+ return emmc_phy_parse_param_dt(host, np, priv->phy_params);
+}
+
+int xenon_phy_parse_dt(struct device_node *np, struct sdhci_host *host)
+{
+ const char *phy_type = NULL;
+
+ if (!of_property_read_string(np, "marvell,xenon-phy-type", &phy_type))
+ return add_xenon_phy(np, host, phy_type);
+
+ dev_info(mmc_dev(host->mmc), "Fail to get Xenon PHY type. Use default eMMC 5.1 PHY\n");
+ return add_xenon_phy(np, host, "emmc 5.1 phy");
+}
diff --git a/drivers/mmc/host/sdhci-xenon.c b/drivers/mmc/host/sdhci-xenon.c
index 0e0c60892f72..e51079b2d01c 100644
--- a/drivers/mmc/host/sdhci-xenon.c
+++ b/drivers/mmc/host/sdhci-xenon.c
@@ -257,6 +257,7 @@ static void xenon_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
spin_unlock_irqrestore(&host->lock, flags);
sdhci_set_ios(mmc, ios);
+ xenon_phy_adj(host, ios);
if (host->clock > XENON_DEFAULT_SDCLK_FREQ) {
spin_lock_irqsave(&host->lock, flags);
@@ -502,7 +503,7 @@ static int xenon_probe_dt(struct platform_device *pdev)
}
priv->tuning_count = tuning_count;
- return err;
+ return xenon_phy_parse_dt(np, host);
}
static int xenon_sdhc_probe(struct sdhci_host *host)
diff --git a/drivers/mmc/host/sdhci-xenon.h b/drivers/mmc/host/sdhci-xenon.h
index 69de711db9eb..e87639774bca 100644
--- a/drivers/mmc/host/sdhci-xenon.h
+++ b/drivers/mmc/host/sdhci-xenon.h
@@ -24,7 +24,17 @@
#define XENON_SYS_EXT_OP_CTRL 0x010C
#define XENON_MASK_CMD_CONFLICT_ERR BIT(8)
+#define XENON_SLOT_OP_STATUS_CTRL 0x0128
+
+#define XENON_TUN_CONSECUTIVE_TIMES_SHIFT 16
+#define XENON_TUN_CONSECUTIVE_TIMES_MASK 0x7
+#define XENON_TUN_CONSECUTIVE_TIMES 0x4
+#define XENON_TUNING_STEP_SHIFT 12
+#define XENON_TUNING_STEP_MASK 0xF
+#define XENON_TUNING_STEP_DIVIDER BIT(6)
+
#define XENON_SLOT_EMMC_CTRL 0x0130
+#define XENON_ENABLE_DATA_STROBE BIT(24)
#define XENON_EMMC_VCCQ_MASK 0x3
#define XENON_EMMC_VCCQ_1_8V 0x1
#define XENON_EMMC_VCCQ_3_3V 0x3
@@ -33,11 +43,17 @@
/* retuning compatible */
#define XENON_RETUNING_COMPATIBLE 0x1
+#define XENON_SLOT_EXT_PRESENT_STATE 0x014C
+#define XENON_DLL_LOCK_STATE 0x1
+
+#define XENON_SLOT_DLL_CUR_DLY_VAL 0x0150
+
/* Tuning Parameter */
#define XENON_TMR_RETUN_NO_PRESENT 0xF
#define XENON_DEF_TUNING_COUNT 0x9
#define XENON_DEFAULT_SDCLK_FREQ 400000
+#define XENON_LOWEST_SDCLK_FREQ 100000
/* Xenon specific Mode Select value */
#define XENON_CTRL_HS200 0x5
@@ -65,6 +81,27 @@ struct sdhci_xenon_priv {
* initialization completes.
*/
unsigned int init_card_type;
+
+ /*
+ * The bus_width, timing, and clock fields in below
+ * record the current ios setting of Xenon SDHC.
+ * Driver will adjust PHY setting if any change to
+ * ios affects PHY timing.
+ */
+ unsigned char bus_width;
+ unsigned char timing;
+ unsigned int clock;
+
+ int phy_type;
+ /*
+ * Contains board-specific PHY parameters
+ * passed from device tree.
+ */
+ void *phy_params;
+ struct xenon_emmc_phy_regs *emmc_phy_regs;
};
+int xenon_phy_adj(struct sdhci_host *host, struct mmc_ios *ios);
+int xenon_phy_parse_dt(struct device_node *np,
+ struct sdhci_host *host);
#endif
--
git-series 0.9.1
^ permalink raw reply related
* [PATCH v5 08/12] mmc: sdhci-xenon: Add SoC PHY PAD voltage control
From: Gregory CLEMENT @ 2017-01-11 17:19 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <cover.8527229dae6c124f24c5e637430c2cdc86f80392.1484154449.git-series.gregory.clement@free-electrons.com>
From: Hu Ziji <huziji@marvell.com>
Some SoCs have PHY PAD outside Xenon IP.
PHY PAD voltage should match signalling voltage in use.
Add generic SoC PHY PAD voltage control interface.
Implement Aramda-3700 SoC PHY PAD voltage control.
Signed-off-by: Hu Ziji <huziji@marvell.com>
Tested-by: Russell King <rmk+kernel@armlinux.org.uk>
Signed-off-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
---
drivers/mmc/host/sdhci-xenon-phy.c | 116 +++++++++++++++++++++++++++++-
drivers/mmc/host/sdhci-xenon.c | 2 +-
drivers/mmc/host/sdhci-xenon.h | 2 +-
3 files changed, 119 insertions(+), 1 deletion(-)
diff --git a/drivers/mmc/host/sdhci-xenon-phy.c b/drivers/mmc/host/sdhci-xenon-phy.c
index f410712d24ea..1221754b7a3d 100644
--- a/drivers/mmc/host/sdhci-xenon-phy.c
+++ b/drivers/mmc/host/sdhci-xenon-phy.c
@@ -143,6 +143,27 @@ enum phy_type_enum {
NR_PHY_TYPES
};
+enum soc_pad_ctrl_type {
+ SOC_PAD_SD,
+ SOC_PAD_FIXED_1_8V,
+};
+
+struct soc_pad_ctrl_table {
+ const char *soc;
+ void (*set_soc_pad)(struct sdhci_host *host,
+ unsigned char signal_voltage);
+};
+
+struct soc_pad_ctrl {
+ /* Register address of SOC PHY PAD ctrl */
+ void __iomem *reg;
+ /* SOC PHY PAD ctrl type */
+ enum soc_pad_ctrl_type pad_type;
+ /* SOC specific operation to set SOC PHY PAD */
+ void (*set_soc_pad)(struct sdhci_host *host,
+ unsigned char signal_voltage);
+};
+
static struct xenon_emmc_phy_regs xenon_emmc_5_0_phy_regs = {
.timing_adj = XENON_EMMC_5_0_PHY_TIMING_ADJUST,
.func_ctrl = XENON_EMMC_5_0_PHY_FUNC_CONTROL,
@@ -176,6 +197,8 @@ struct emmc_phy_params {
u8 nr_tun_times;
/* Divider for calculating Tuning Step */
u8 tun_step_divider;
+
+ struct soc_pad_ctrl pad_ctrl;
};
static int alloc_emmc_phy(struct sdhci_xenon_priv *priv)
@@ -252,6 +275,45 @@ static int emmc_phy_init(struct sdhci_host *host)
return 0;
}
+#define ARMADA_3700_SOC_PAD_1_8V 0x1
+#define ARMADA_3700_SOC_PAD_3_3V 0x0
+
+static void armada_3700_soc_pad_voltage_set(struct sdhci_host *host,
+ unsigned char signal_voltage)
+{
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ struct emmc_phy_params *params = priv->phy_params;
+
+ if (params->pad_ctrl.pad_type == SOC_PAD_FIXED_1_8V) {
+ writel(ARMADA_3700_SOC_PAD_1_8V, params->pad_ctrl.reg);
+ } else if (params->pad_ctrl.pad_type == SOC_PAD_SD) {
+ if (signal_voltage == MMC_SIGNAL_VOLTAGE_180)
+ writel(ARMADA_3700_SOC_PAD_1_8V, params->pad_ctrl.reg);
+ else if (signal_voltage == MMC_SIGNAL_VOLTAGE_330)
+ writel(ARMADA_3700_SOC_PAD_3_3V, params->pad_ctrl.reg);
+ }
+}
+
+/*
+ * Set SOC PHY voltage PAD control register,
+ * according to the operation voltage on PAD.
+ * The detailed operation depends on SOC implementaion.
+ */
+static void emmc_phy_set_soc_pad(struct sdhci_host *host,
+ unsigned char signal_voltage)
+{
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+ struct emmc_phy_params *params = priv->phy_params;
+
+ if (!params->pad_ctrl.reg)
+ return;
+
+ if (params->pad_ctrl.set_soc_pad)
+ params->pad_ctrl.set_soc_pad(host, signal_voltage);
+}
+
/*
* Enable eMMC PHY HW DLL
* DLL should be enabled and stable before HS200/SDR104 tuning,
@@ -621,6 +683,51 @@ static void emmc_phy_set(struct sdhci_host *host,
dev_dbg(mmc_dev(host->mmc), "eMMC PHY setting completes\n");
}
+static int get_dt_pad_ctrl_data(struct sdhci_host *host,
+ struct device_node *np,
+ struct emmc_phy_params *params)
+{
+ int ret = 0;
+ const char *name;
+ struct resource iomem;
+
+ if (of_device_is_compatible(np, "marvell,armada-3700-sdhci"))
+ params->pad_ctrl.set_soc_pad = armada_3700_soc_pad_voltage_set;
+ else
+ return 0;
+
+ if (of_address_to_resource(np, 1, &iomem)) {
+ dev_err(mmc_dev(host->mmc), "Unable to find SOC PAD ctrl register address for %s\n",
+ np->name);
+ return -EINVAL;
+ }
+
+ params->pad_ctrl.reg = devm_ioremap_resource(mmc_dev(host->mmc),
+ &iomem);
+ if (IS_ERR(params->pad_ctrl.reg)) {
+ dev_err(mmc_dev(host->mmc), "Unable to get SOC PHY PAD ctrl regiser for %s\n",
+ np->name);
+ return PTR_ERR(params->pad_ctrl.reg);
+ }
+
+ ret = of_property_read_string(np, "marvell,pad-type", &name);
+ if (ret) {
+ dev_err(mmc_dev(host->mmc), "Unable to determine SOC PHY PAD ctrl type\n");
+ return ret;
+ }
+ if (!strcmp(name, "sd")) {
+ params->pad_ctrl.pad_type = SOC_PAD_SD;
+ } else if (!strcmp(name, "fixed-1-8v")) {
+ params->pad_ctrl.pad_type = SOC_PAD_FIXED_1_8V;
+ } else {
+ dev_err(mmc_dev(host->mmc), "Unsupported SOC PHY PAD ctrl type %s\n",
+ name);
+ return -EINVAL;
+ }
+
+ return ret;
+}
+
static int emmc_phy_parse_param_dt(struct sdhci_host *host,
struct device_node *np,
struct emmc_phy_params *params)
@@ -654,7 +761,14 @@ static int emmc_phy_parse_param_dt(struct sdhci_host *host,
else
params->tun_step_divider = XENON_TUNING_STEP_DIVIDER;
- return 0;
+ return get_dt_pad_ctrl_data(host, np, params);
+}
+
+/* Set SOC PHY Voltage PAD */
+void xenon_soc_pad_ctrl(struct sdhci_host *host,
+ unsigned char signal_voltage)
+{
+ emmc_phy_set_soc_pad(host, signal_voltage);
}
/*
diff --git a/drivers/mmc/host/sdhci-xenon.c b/drivers/mmc/host/sdhci-xenon.c
index e51079b2d01c..be93af56c348 100644
--- a/drivers/mmc/host/sdhci-xenon.c
+++ b/drivers/mmc/host/sdhci-xenon.c
@@ -335,6 +335,8 @@ static int xenon_start_signal_voltage_switch(struct mmc_host *mmc,
*/
enable_xenon_internal_clk(host);
+ xenon_soc_pad_ctrl(host, ios->signal_voltage);
+
if (priv->init_card_type == MMC_TYPE_MMC)
return xenon_emmc_signal_voltage_switch(mmc, ios);
diff --git a/drivers/mmc/host/sdhci-xenon.h b/drivers/mmc/host/sdhci-xenon.h
index e87639774bca..57e7cb463413 100644
--- a/drivers/mmc/host/sdhci-xenon.h
+++ b/drivers/mmc/host/sdhci-xenon.h
@@ -104,4 +104,6 @@ struct sdhci_xenon_priv {
int xenon_phy_adj(struct sdhci_host *host, struct mmc_ios *ios);
int xenon_phy_parse_dt(struct device_node *np,
struct sdhci_host *host);
+void xenon_soc_pad_ctrl(struct sdhci_host *host,
+ unsigned char signal_voltage);
#endif
--
git-series 0.9.1
^ permalink raw reply related
* [PATCH v5 09/12] MAINTAINERS: add entry for Marvell Xenon MMC Host Controller drivers
From: Gregory CLEMENT @ 2017-01-11 17:19 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <cover.8527229dae6c124f24c5e637430c2cdc86f80392.1484154449.git-series.gregory.clement@free-electrons.com>
From: Hu Ziji <huziji@marvell.com>
Add maintainer entry for Marvell Xenon eMMC/SD/SDIO
Host Controller drivers.
Signed-off-by: Hu Ziji <huziji@marvell.com>
Signed-off-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
---
MAINTAINERS | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index cfff2c9e3d94..f4fea77165d5 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -7757,6 +7757,13 @@ M: Nicolas Pitre <nico@fluxnic.net>
S: Odd Fixes
F: drivers/mmc/host/mvsdio.*
+MARVELL XENON MMC/SD/SDIO HOST CONTROLLER DRIVER
+M: Ziji Hu <huziji@marvell.com>
+L: linux-mmc at vger.kernel.org
+S: Supported
+F: drivers/mmc/host/sdhci-xenon*
+F: Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt
+
MATROX FRAMEBUFFER DRIVER
L: linux-fbdev at vger.kernel.org
S: Orphan
--
git-series 0.9.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox