From: Ard Biesheuvel <ard.biesheuvel@linaro.org>
To: linux-crypto@vger.kernel.org
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>,
Herbert Xu <herbert@gondor.apana.org.au>,
Eric Biggers <ebiggers@google.com>,
Steve Capper <steve.capper@arm.com>,
Ondrej Mosnacek <omosnace@redhat.com>,
linux-arm-kernel@lists.infradead.org
Subject: [PATCH 4/6] crypto: aegis128 - add support for SIMD acceleration
Date: Mon, 24 Jun 2019 09:38:16 +0200 [thread overview]
Message-ID: <20190624073818.29296-5-ard.biesheuvel@linaro.org> (raw)
In-Reply-To: <20190624073818.29296-1-ard.biesheuvel@linaro.org>
Add some plumbing to allow the AEGIS128 code to be built with SIMD
routines for acceleration.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
crypto/aegis.h | 14 +++----
crypto/aegis128.c | 42 ++++++++++++++++++--
2 files changed, 45 insertions(+), 11 deletions(-)
diff --git a/crypto/aegis.h b/crypto/aegis.h
index 3308066ddde0..6cb65a497ba2 100644
--- a/crypto/aegis.h
+++ b/crypto/aegis.h
@@ -35,23 +35,23 @@ static const union aegis_block crypto_aegis_const[2] = {
} },
};
-static void crypto_aegis_block_xor(union aegis_block *dst,
- const union aegis_block *src)
+static inline void crypto_aegis_block_xor(union aegis_block *dst,
+ const union aegis_block *src)
{
dst->words64[0] ^= src->words64[0];
dst->words64[1] ^= src->words64[1];
}
-static void crypto_aegis_block_and(union aegis_block *dst,
- const union aegis_block *src)
+static inline void crypto_aegis_block_and(union aegis_block *dst,
+ const union aegis_block *src)
{
dst->words64[0] &= src->words64[0];
dst->words64[1] &= src->words64[1];
}
-static void crypto_aegis_aesenc(union aegis_block *dst,
- const union aegis_block *src,
- const union aegis_block *key)
+static inline void crypto_aegis_aesenc(union aegis_block *dst,
+ const union aegis_block *src,
+ const union aegis_block *key)
{
const u8 *s = src->bytes;
const u32 *t = crypto_ft_tab[0];
diff --git a/crypto/aegis128.c b/crypto/aegis128.c
index 4f8f1cdef129..1bbd3e49c890 100644
--- a/crypto/aegis128.c
+++ b/crypto/aegis128.c
@@ -8,6 +8,7 @@
#include <crypto/algapi.h>
#include <crypto/internal/aead.h>
+#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <linux/err.h>
@@ -15,6 +16,7 @@
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
+#include <asm/simd.h>
#include "aegis.h"
@@ -40,6 +42,15 @@ struct aegis128_ops {
const u8 *src, unsigned int size);
};
+static bool have_simd;
+
+bool crypto_aegis128_have_simd(void);
+void crypto_aegis128_update_simd(struct aegis_state *state, const void *msg);
+void crypto_aegis128_encrypt_chunk_simd(struct aegis_state *state, u8 *dst,
+ const u8 *src, unsigned int size);
+void crypto_aegis128_decrypt_chunk_simd(struct aegis_state *state, u8 *dst,
+ const u8 *src, unsigned int size);
+
static void crypto_aegis128_update(struct aegis_state *state)
{
union aegis_block tmp;
@@ -55,12 +66,22 @@ static void crypto_aegis128_update(struct aegis_state *state)
static void crypto_aegis128_update_a(struct aegis_state *state,
const union aegis_block *msg)
{
+ if (have_simd && crypto_simd_usable()) {
+ crypto_aegis128_update_simd(state, msg);
+ return;
+ }
+
crypto_aegis128_update(state);
crypto_aegis_block_xor(&state->blocks[0], msg);
}
static void crypto_aegis128_update_u(struct aegis_state *state, const void *msg)
{
+ if (have_simd && crypto_simd_usable()) {
+ crypto_aegis128_update_simd(state, msg);
+ return;
+ }
+
crypto_aegis128_update(state);
crypto_xor(state->blocks[0].bytes, msg, AEGIS_BLOCK_SIZE);
}
@@ -365,7 +386,7 @@ static void crypto_aegis128_crypt(struct aead_request *req,
static int crypto_aegis128_encrypt(struct aead_request *req)
{
- static const struct aegis128_ops ops = {
+ const struct aegis128_ops *ops = &(struct aegis128_ops){
.skcipher_walk_init = skcipher_walk_aead_encrypt,
.crypt_chunk = crypto_aegis128_encrypt_chunk,
};
@@ -375,7 +396,12 @@ static int crypto_aegis128_encrypt(struct aead_request *req)
unsigned int authsize = crypto_aead_authsize(tfm);
unsigned int cryptlen = req->cryptlen;
- crypto_aegis128_crypt(req, &tag, cryptlen, &ops);
+ if (have_simd && crypto_simd_usable())
+ ops = &(struct aegis128_ops){
+ .skcipher_walk_init = skcipher_walk_aead_encrypt,
+ .crypt_chunk = crypto_aegis128_encrypt_chunk_simd };
+
+ crypto_aegis128_crypt(req, &tag, cryptlen, ops);
scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen,
authsize, 1);
@@ -384,7 +410,7 @@ static int crypto_aegis128_encrypt(struct aead_request *req)
static int crypto_aegis128_decrypt(struct aead_request *req)
{
- static const struct aegis128_ops ops = {
+ const struct aegis128_ops *ops = &(struct aegis128_ops){
.skcipher_walk_init = skcipher_walk_aead_decrypt,
.crypt_chunk = crypto_aegis128_decrypt_chunk,
};
@@ -398,7 +424,12 @@ static int crypto_aegis128_decrypt(struct aead_request *req)
scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen,
authsize, 0);
- crypto_aegis128_crypt(req, &tag, cryptlen, &ops);
+ if (have_simd && crypto_simd_usable())
+ ops = &(struct aegis128_ops){
+ .skcipher_walk_init = skcipher_walk_aead_decrypt,
+ .crypt_chunk = crypto_aegis128_decrypt_chunk_simd };
+
+ crypto_aegis128_crypt(req, &tag, cryptlen, ops);
return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0;
}
@@ -429,6 +460,9 @@ static struct aead_alg crypto_aegis128_alg = {
static int __init crypto_aegis128_module_init(void)
{
+ if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD))
+ have_simd = crypto_aegis128_have_simd();
+
return crypto_register_aead(&crypto_aegis128_alg);
}
--
2.20.1
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
next prev parent reply other threads:[~2019-06-24 7:40 UTC|newest]
Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-06-24 7:38 [PATCH 0/6] crypto: aegis128 - add NEON intrinsics version for ARM/arm64 Ard Biesheuvel
2019-06-24 7:38 ` [PATCH 1/6] crypto: aegis128 - use unaliged helper in unaligned decrypt path Ard Biesheuvel
2019-06-24 7:59 ` Ondrej Mosnacek
2019-06-24 8:01 ` Ard Biesheuvel
2019-06-24 7:38 ` [PATCH 2/6] crypto: aegis - drop empty TFM init/exit routines Ard Biesheuvel
2019-06-24 8:03 ` Ondrej Mosnacek
2019-06-24 7:38 ` [PATCH 3/6] crypto: aegis - avoid prerotated AES tables Ard Biesheuvel
2019-06-24 8:13 ` Ondrej Mosnacek
2019-06-24 7:38 ` Ard Biesheuvel [this message]
2019-06-24 7:38 ` [PATCH 5/6] crypto: aegis128 - provide a SIMD implementation based on NEON intrinsics Ard Biesheuvel
2019-06-24 14:37 ` Ard Biesheuvel
2019-06-24 7:38 ` [PATCH 6/6] crypto: tcrypt - add a speed test for AEGIS128 Ard Biesheuvel
2019-06-24 16:56 ` [PATCH 0/6] crypto: aegis128 - add NEON intrinsics version for ARM/arm64 Eric Biggers
2019-06-25 14:07 ` Ondrej Mosnacek
2019-06-25 14:57 ` Ard Biesheuvel
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190624073818.29296-5-ard.biesheuvel@linaro.org \
--to=ard.biesheuvel@linaro.org \
--cc=ebiggers@google.com \
--cc=herbert@gondor.apana.org.au \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-crypto@vger.kernel.org \
--cc=omosnace@redhat.com \
--cc=steve.capper@arm.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).