[PATCH] crypto: aead - un-inline encrypt and decrypt functions

[PATCH] crypto: aead - un-inline encrypt and decrypt functions

[Eric Biggers]

From: Eric Biggers <ebiggers@google.com>

crypto_aead_encrypt() and crypto_aead_decrypt() have grown to be more
than a single indirect function call.  They now also check whether a key
has been set, the decryption side checks whether the input is at least
as long as the authentication tag length, and with CONFIG_CRYPTO_STATS=y
they also update the crypto statistics.  That can add up to a lot of
bloat at every call site.  Moreover, these always involve a function
call anyway, which greatly limits the benefits of inlining.

So change them to be non-inline.

Signed-off-by: Eric Biggers <ebiggers@google.com>
---
 crypto/aead.c         | 36 ++++++++++++++++++++++++++++++++++++
 include/crypto/aead.h | 34 ++--------------------------------
 2 files changed, 38 insertions(+), 32 deletions(-)

diff --git a/crypto/aead.c b/crypto/aead.c
index 4908b5e846f0e..fc1d7ad8a487d 100644
--- a/crypto/aead.c
+++ b/crypto/aead.c
@@ -89,6 +89,42 @@ int crypto_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
 }
 EXPORT_SYMBOL_GPL(crypto_aead_setauthsize);
 
+int crypto_aead_encrypt(struct aead_request *req)
+{
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct crypto_alg *alg = aead->base.__crt_alg;
+	unsigned int cryptlen = req->cryptlen;
+	int ret;
+
+	crypto_stats_get(alg);
+	if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
+		ret = -ENOKEY;
+	else
+		ret = crypto_aead_alg(aead)->encrypt(req);
+	crypto_stats_aead_encrypt(cryptlen, alg, ret);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_aead_encrypt);
+
+int crypto_aead_decrypt(struct aead_request *req)
+{
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct crypto_alg *alg = aead->base.__crt_alg;
+	unsigned int cryptlen = req->cryptlen;
+	int ret;
+
+	crypto_stats_get(alg);
+	if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
+		ret = -ENOKEY;
+	else if (req->cryptlen < crypto_aead_authsize(aead))
+		ret = -EINVAL;
+	else
+		ret = crypto_aead_alg(aead)->decrypt(req);
+	crypto_stats_aead_decrypt(cryptlen, alg, ret);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_aead_decrypt);
+
 static void crypto_aead_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_aead *aead = __crypto_aead_cast(tfm);
diff --git a/include/crypto/aead.h b/include/crypto/aead.h
index 9ad595f97c65a..020d581373abd 100644
--- a/include/crypto/aead.h
+++ b/include/crypto/aead.h
@@ -322,21 +322,7 @@ static inline struct crypto_aead *crypto_aead_reqtfm(struct aead_request *req)
  *
  * Return: 0 if the cipher operation was successful; < 0 if an error occurred
  */
-static inline int crypto_aead_encrypt(struct aead_request *req)
-{
-	struct crypto_aead *aead = crypto_aead_reqtfm(req);
-	struct crypto_alg *alg = aead->base.__crt_alg;
-	unsigned int cryptlen = req->cryptlen;
-	int ret;
-
-	crypto_stats_get(alg);
-	if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
-		ret = -ENOKEY;
-	else
-		ret = crypto_aead_alg(aead)->encrypt(req);
-	crypto_stats_aead_encrypt(cryptlen, alg, ret);
-	return ret;
-}
+int crypto_aead_encrypt(struct aead_request *req);
 
 /**
  * crypto_aead_decrypt() - decrypt ciphertext
@@ -360,23 +346,7 @@ static inline int crypto_aead_encrypt(struct aead_request *req)
  *	   integrity of the ciphertext or the associated data was violated);
  *	   < 0 if an error occurred.
  */
-static inline int crypto_aead_decrypt(struct aead_request *req)
-{
-	struct crypto_aead *aead = crypto_aead_reqtfm(req);
-	struct crypto_alg *alg = aead->base.__crt_alg;
-	unsigned int cryptlen = req->cryptlen;
-	int ret;
-
-	crypto_stats_get(alg);
-	if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
-		ret = -ENOKEY;
-	else if (req->cryptlen < crypto_aead_authsize(aead))
-		ret = -EINVAL;
-	else
-		ret = crypto_aead_alg(aead)->decrypt(req);
-	crypto_stats_aead_decrypt(cryptlen, alg, ret);
-	return ret;
-}
+int crypto_aead_decrypt(struct aead_request *req);
 
 /**
  * DOC: Asynchronous AEAD Request Handle
-- 
2.21.0

Re: [PATCH] crypto: aead - un-inline encrypt and decrypt functions

[Ard Biesheuvel]

On Mon, 3 Jun 2019 at 07:45, Eric Biggers <ebiggers@kernel.org> wrote:
>
> From: Eric Biggers <ebiggers@google.com>
>
> crypto_aead_encrypt() and crypto_aead_decrypt() have grown to be more
> than a single indirect function call.  They now also check whether a key
> has been set, the decryption side checks whether the input is at least
> as long as the authentication tag length, and with CONFIG_CRYPTO_STATS=y
> they also update the crypto statistics.  That can add up to a lot of
> bloat at every call site.  Moreover, these always involve a function
> call anyway, which greatly limits the benefits of inlining.
>
> So change them to be non-inline.
>
> Signed-off-by: Eric Biggers <ebiggers@google.com>

Acked-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>

> ---
>  crypto/aead.c         | 36 ++++++++++++++++++++++++++++++++++++
>  include/crypto/aead.h | 34 ++--------------------------------
>  2 files changed, 38 insertions(+), 32 deletions(-)
>
> diff --git a/crypto/aead.c b/crypto/aead.c
> index 4908b5e846f0e..fc1d7ad8a487d 100644
> --- a/crypto/aead.c
> +++ b/crypto/aead.c
> @@ -89,6 +89,42 @@ int crypto_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
>  }
>  EXPORT_SYMBOL_GPL(crypto_aead_setauthsize);
>
> +int crypto_aead_encrypt(struct aead_request *req)
> +{
> +       struct crypto_aead *aead = crypto_aead_reqtfm(req);
> +       struct crypto_alg *alg = aead->base.__crt_alg;
> +       unsigned int cryptlen = req->cryptlen;
> +       int ret;
> +
> +       crypto_stats_get(alg);
> +       if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
> +               ret = -ENOKEY;
> +       else
> +               ret = crypto_aead_alg(aead)->encrypt(req);
> +       crypto_stats_aead_encrypt(cryptlen, alg, ret);
> +       return ret;
> +}
> +EXPORT_SYMBOL_GPL(crypto_aead_encrypt);
> +
> +int crypto_aead_decrypt(struct aead_request *req)
> +{
> +       struct crypto_aead *aead = crypto_aead_reqtfm(req);
> +       struct crypto_alg *alg = aead->base.__crt_alg;
> +       unsigned int cryptlen = req->cryptlen;
> +       int ret;
> +
> +       crypto_stats_get(alg);
> +       if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
> +               ret = -ENOKEY;
> +       else if (req->cryptlen < crypto_aead_authsize(aead))
> +               ret = -EINVAL;
> +       else
> +               ret = crypto_aead_alg(aead)->decrypt(req);
> +       crypto_stats_aead_decrypt(cryptlen, alg, ret);
> +       return ret;
> +}
> +EXPORT_SYMBOL_GPL(crypto_aead_decrypt);
> +
>  static void crypto_aead_exit_tfm(struct crypto_tfm *tfm)
>  {
>         struct crypto_aead *aead = __crypto_aead_cast(tfm);
> diff --git a/include/crypto/aead.h b/include/crypto/aead.h
> index 9ad595f97c65a..020d581373abd 100644
> --- a/include/crypto/aead.h
> +++ b/include/crypto/aead.h
> @@ -322,21 +322,7 @@ static inline struct crypto_aead *crypto_aead_reqtfm(struct aead_request *req)
>   *
>   * Return: 0 if the cipher operation was successful; < 0 if an error occurred
>   */
> -static inline int crypto_aead_encrypt(struct aead_request *req)
> -{
> -       struct crypto_aead *aead = crypto_aead_reqtfm(req);
> -       struct crypto_alg *alg = aead->base.__crt_alg;
> -       unsigned int cryptlen = req->cryptlen;
> -       int ret;
> -
> -       crypto_stats_get(alg);
> -       if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
> -               ret = -ENOKEY;
> -       else
> -               ret = crypto_aead_alg(aead)->encrypt(req);
> -       crypto_stats_aead_encrypt(cryptlen, alg, ret);
> -       return ret;
> -}
> +int crypto_aead_encrypt(struct aead_request *req);
>
>  /**
>   * crypto_aead_decrypt() - decrypt ciphertext
> @@ -360,23 +346,7 @@ static inline int crypto_aead_encrypt(struct aead_request *req)
>   *        integrity of the ciphertext or the associated data was violated);
>   *        < 0 if an error occurred.
>   */
> -static inline int crypto_aead_decrypt(struct aead_request *req)
> -{
> -       struct crypto_aead *aead = crypto_aead_reqtfm(req);
> -       struct crypto_alg *alg = aead->base.__crt_alg;
> -       unsigned int cryptlen = req->cryptlen;
> -       int ret;
> -
> -       crypto_stats_get(alg);
> -       if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
> -               ret = -ENOKEY;
> -       else if (req->cryptlen < crypto_aead_authsize(aead))
> -               ret = -EINVAL;
> -       else
> -               ret = crypto_aead_alg(aead)->decrypt(req);
> -       crypto_stats_aead_decrypt(cryptlen, alg, ret);
> -       return ret;
> -}
> +int crypto_aead_decrypt(struct aead_request *req);
>
>  /**
>   * DOC: Asynchronous AEAD Request Handle
> --
> 2.21.0
>

Re: [PATCH] crypto: aead - un-inline encrypt and decrypt functions

[Herbert Xu]

Eric Biggers <ebiggers@kernel.org> wrote:
> From: Eric Biggers <ebiggers@google.com>
> 
> crypto_aead_encrypt() and crypto_aead_decrypt() have grown to be more
> than a single indirect function call.  They now also check whether a key
> has been set, the decryption side checks whether the input is at least
> as long as the authentication tag length, and with CONFIG_CRYPTO_STATS=y
> they also update the crypto statistics.  That can add up to a lot of
> bloat at every call site.  Moreover, these always involve a function
> call anyway, which greatly limits the benefits of inlining.
> 
> So change them to be non-inline.
> 
> Signed-off-by: Eric Biggers <ebiggers@google.com>
> ---
> crypto/aead.c         | 36 ++++++++++++++++++++++++++++++++++++
> include/crypto/aead.h | 34 ++--------------------------------
> 2 files changed, 38 insertions(+), 32 deletions(-)

Patch applied.  Thanks.
-- 
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt