* [PATCH v4] crypto: gf128mul - define gf128mul_x_* in gf128mul.h
From: Ondrej Mosnacek @ 2017-04-01 15:17 UTC (permalink / raw)
To: Herbert Xu
Cc: David S. Miller, linux-crypto, Jeffrey Walton, Milan Broz,
Ondrej Mosnacek, Eric Biggers
The gf128mul_x_ble function is currently defined in gf128mul.c, because
it depends on the gf128mul_table_be multiplication table.
However, since the function is very small and only uses two values from
the table, it is better for it to be defined as inline function in
gf128mul.h. That way, the function can be inlined by the compiler for
better performance.
For consistency, the other gf128mul_x_* functions are also moved to the
header file. In addition, the code is rewritten to be constant-time.
After this change, the speed of the generic 'xts(aes)' implementation
increased from ~225 MiB/s to ~235 MiB/s (measured using 'cryptsetup
benchmark -c aes-xts-plain64' on an Intel system with CRYPTO_AES_X86_64
and CRYPTO_AES_NI_INTEL disabled).
Signed-off-by: Ondrej Mosnacek <omosnacek@gmail.com>
Cc: Eric Biggers <ebiggers@google.com>
---
v3 -> v4: a faster version of gf128mul_x_lle
v2 -> v3: constant-time implementation
v1 -> v2: move all _x_ functions to the header, not just gf128mul_x_ble
crypto/gf128mul.c | 33 +---------------------------
include/crypto/gf128mul.h | 55 +++++++++++++++++++++++++++++++++++++++++++++--
2 files changed, 54 insertions(+), 34 deletions(-)
diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c
index 04facc0..dc01212 100644
--- a/crypto/gf128mul.c
+++ b/crypto/gf128mul.c
@@ -130,43 +130,12 @@ static const u16 gf128mul_table_le[256] = gf128mul_dat(xda_le);
static const u16 gf128mul_table_be[256] = gf128mul_dat(xda_be);
/*
- * The following functions multiply a field element by x or by x^8 in
+ * The following functions multiply a field element by x^8 in
* the polynomial field representation. They use 64-bit word operations
* to gain speed but compensate for machine endianness and hence work
* correctly on both styles of machine.
*/
-static void gf128mul_x_lle(be128 *r, const be128 *x)
-{
- u64 a = be64_to_cpu(x->a);
- u64 b = be64_to_cpu(x->b);
- u64 _tt = gf128mul_table_le[(b << 7) & 0xff];
-
- r->b = cpu_to_be64((b >> 1) | (a << 63));
- r->a = cpu_to_be64((a >> 1) ^ (_tt << 48));
-}
-
-static void gf128mul_x_bbe(be128 *r, const be128 *x)
-{
- u64 a = be64_to_cpu(x->a);
- u64 b = be64_to_cpu(x->b);
- u64 _tt = gf128mul_table_be[a >> 63];
-
- r->a = cpu_to_be64((a << 1) | (b >> 63));
- r->b = cpu_to_be64((b << 1) ^ _tt);
-}
-
-void gf128mul_x_ble(be128 *r, const be128 *x)
-{
- u64 a = le64_to_cpu(x->a);
- u64 b = le64_to_cpu(x->b);
- u64 _tt = gf128mul_table_be[b >> 63];
-
- r->a = cpu_to_le64((a << 1) ^ _tt);
- r->b = cpu_to_le64((b << 1) | (a >> 63));
-}
-EXPORT_SYMBOL(gf128mul_x_ble);
-
static void gf128mul_x8_lle(be128 *x)
{
u64 a = be64_to_cpu(x->a);
diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h
index 0bc9b5f..35ced9d 100644
--- a/include/crypto/gf128mul.h
+++ b/include/crypto/gf128mul.h
@@ -49,6 +49,7 @@
#ifndef _CRYPTO_GF128MUL_H
#define _CRYPTO_GF128MUL_H
+#include <asm/byteorder.h>
#include <crypto/b128ops.h>
#include <linux/slab.h>
@@ -163,8 +164,58 @@ void gf128mul_lle(be128 *a, const be128 *b);
void gf128mul_bbe(be128 *a, const be128 *b);
-/* multiply by x in ble format, needed by XTS */
-void gf128mul_x_ble(be128 *a, const be128 *b);
+/*
+ * The following functions multiply a field element by x in
+ * the polynomial field representation. They use 64-bit word operations
+ * to gain speed but compensate for machine endianness and hence work
+ * correctly on both styles of machine.
+ *
+ * They are defined here for performance.
+ */
+
+static inline u64 gf128mul_mask_from_bit(u64 x, int which)
+{
+ /* a constant-time version of 'x & ((u64)1 << which) ? (u64)-1 : 0' */
+ return ((s64)(x << (63 - which)) >> 63);
+}
+
+static inline void gf128mul_x_lle(be128 *r, const be128 *x)
+{
+ u64 a = be64_to_cpu(x->a);
+ u64 b = be64_to_cpu(x->b);
+
+ /* equivalent to gf128mul_table_le[(b << 7) & 0xff] << 48
+ * (see crypto/gf128mul.c): */
+ u64 _tt = gf128mul_mask_from_bit(b, 0) & ((u64)0xe1 << 56);
+
+ r->b = cpu_to_be64((b >> 1) | (a << 63));
+ r->a = cpu_to_be64((a >> 1) ^ _tt);
+}
+
+static inline void gf128mul_x_bbe(be128 *r, const be128 *x)
+{
+ u64 a = be64_to_cpu(x->a);
+ u64 b = be64_to_cpu(x->b);
+
+ /* equivalent to gf128mul_table_be[a >> 63] (see crypto/gf128mul.c): */
+ u64 _tt = gf128mul_mask_from_bit(a, 63) & 0x87;
+
+ r->a = cpu_to_be64((a << 1) | (b >> 63));
+ r->b = cpu_to_be64((b << 1) ^ _tt);
+}
+
+/* needed by XTS */
+static inline void gf128mul_x_ble(be128 *r, const be128 *x)
+{
+ u64 a = le64_to_cpu(x->a);
+ u64 b = le64_to_cpu(x->b);
+
+ /* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */
+ u64 _tt = gf128mul_mask_from_bit(b, 63) & 0x87;
+
+ r->a = cpu_to_le64((a << 1) ^ _tt);
+ r->b = cpu_to_le64((b << 1) | (a >> 63));
+}
/* 4k table optimization */
--
2.9.3
^ permalink raw reply related
* Re: [PATCH v4] crypto: gf128mul - define gf128mul_x_* in gf128mul.h
From: Ondrej Mosnáček @ 2017-04-01 15:19 UTC (permalink / raw)
To: Herbert Xu
Cc: David S. Miller, linux-crypto, Jeffrey Walton, Milan Broz,
Ondrej Mosnacek, Eric Biggers
In-Reply-To: <20170401151755.11875-1-omosnacek@gmail.com>
Oops, sorry, wrong prefix...
2017-04-01 17:17 GMT+02:00 Ondrej Mosnacek <omosnacek@gmail.com>:
> The gf128mul_x_ble function is currently defined in gf128mul.c, because
> it depends on the gf128mul_table_be multiplication table.
>
> However, since the function is very small and only uses two values from
> the table, it is better for it to be defined as inline function in
> gf128mul.h. That way, the function can be inlined by the compiler for
> better performance.
>
> For consistency, the other gf128mul_x_* functions are also moved to the
> header file. In addition, the code is rewritten to be constant-time.
>
> After this change, the speed of the generic 'xts(aes)' implementation
> increased from ~225 MiB/s to ~235 MiB/s (measured using 'cryptsetup
> benchmark -c aes-xts-plain64' on an Intel system with CRYPTO_AES_X86_64
> and CRYPTO_AES_NI_INTEL disabled).
>
> Signed-off-by: Ondrej Mosnacek <omosnacek@gmail.com>
> Cc: Eric Biggers <ebiggers@google.com>
> ---
> v3 -> v4: a faster version of gf128mul_x_lle
> v2 -> v3: constant-time implementation
> v1 -> v2: move all _x_ functions to the header, not just gf128mul_x_ble
>
> crypto/gf128mul.c | 33 +---------------------------
> include/crypto/gf128mul.h | 55 +++++++++++++++++++++++++++++++++++++++++++++--
> 2 files changed, 54 insertions(+), 34 deletions(-)
>
> diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c
> index 04facc0..dc01212 100644
> --- a/crypto/gf128mul.c
> +++ b/crypto/gf128mul.c
> @@ -130,43 +130,12 @@ static const u16 gf128mul_table_le[256] = gf128mul_dat(xda_le);
> static const u16 gf128mul_table_be[256] = gf128mul_dat(xda_be);
>
> /*
> - * The following functions multiply a field element by x or by x^8 in
> + * The following functions multiply a field element by x^8 in
> * the polynomial field representation. They use 64-bit word operations
> * to gain speed but compensate for machine endianness and hence work
> * correctly on both styles of machine.
> */
>
> -static void gf128mul_x_lle(be128 *r, const be128 *x)
> -{
> - u64 a = be64_to_cpu(x->a);
> - u64 b = be64_to_cpu(x->b);
> - u64 _tt = gf128mul_table_le[(b << 7) & 0xff];
> -
> - r->b = cpu_to_be64((b >> 1) | (a << 63));
> - r->a = cpu_to_be64((a >> 1) ^ (_tt << 48));
> -}
> -
> -static void gf128mul_x_bbe(be128 *r, const be128 *x)
> -{
> - u64 a = be64_to_cpu(x->a);
> - u64 b = be64_to_cpu(x->b);
> - u64 _tt = gf128mul_table_be[a >> 63];
> -
> - r->a = cpu_to_be64((a << 1) | (b >> 63));
> - r->b = cpu_to_be64((b << 1) ^ _tt);
> -}
> -
> -void gf128mul_x_ble(be128 *r, const be128 *x)
> -{
> - u64 a = le64_to_cpu(x->a);
> - u64 b = le64_to_cpu(x->b);
> - u64 _tt = gf128mul_table_be[b >> 63];
> -
> - r->a = cpu_to_le64((a << 1) ^ _tt);
> - r->b = cpu_to_le64((b << 1) | (a >> 63));
> -}
> -EXPORT_SYMBOL(gf128mul_x_ble);
> -
> static void gf128mul_x8_lle(be128 *x)
> {
> u64 a = be64_to_cpu(x->a);
> diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h
> index 0bc9b5f..35ced9d 100644
> --- a/include/crypto/gf128mul.h
> +++ b/include/crypto/gf128mul.h
> @@ -49,6 +49,7 @@
> #ifndef _CRYPTO_GF128MUL_H
> #define _CRYPTO_GF128MUL_H
>
> +#include <asm/byteorder.h>
> #include <crypto/b128ops.h>
> #include <linux/slab.h>
>
> @@ -163,8 +164,58 @@ void gf128mul_lle(be128 *a, const be128 *b);
>
> void gf128mul_bbe(be128 *a, const be128 *b);
>
> -/* multiply by x in ble format, needed by XTS */
> -void gf128mul_x_ble(be128 *a, const be128 *b);
> +/*
> + * The following functions multiply a field element by x in
> + * the polynomial field representation. They use 64-bit word operations
> + * to gain speed but compensate for machine endianness and hence work
> + * correctly on both styles of machine.
> + *
> + * They are defined here for performance.
> + */
> +
> +static inline u64 gf128mul_mask_from_bit(u64 x, int which)
> +{
> + /* a constant-time version of 'x & ((u64)1 << which) ? (u64)-1 : 0' */
> + return ((s64)(x << (63 - which)) >> 63);
> +}
> +
> +static inline void gf128mul_x_lle(be128 *r, const be128 *x)
> +{
> + u64 a = be64_to_cpu(x->a);
> + u64 b = be64_to_cpu(x->b);
> +
> + /* equivalent to gf128mul_table_le[(b << 7) & 0xff] << 48
> + * (see crypto/gf128mul.c): */
> + u64 _tt = gf128mul_mask_from_bit(b, 0) & ((u64)0xe1 << 56);
> +
> + r->b = cpu_to_be64((b >> 1) | (a << 63));
> + r->a = cpu_to_be64((a >> 1) ^ _tt);
> +}
> +
> +static inline void gf128mul_x_bbe(be128 *r, const be128 *x)
> +{
> + u64 a = be64_to_cpu(x->a);
> + u64 b = be64_to_cpu(x->b);
> +
> + /* equivalent to gf128mul_table_be[a >> 63] (see crypto/gf128mul.c): */
> + u64 _tt = gf128mul_mask_from_bit(a, 63) & 0x87;
> +
> + r->a = cpu_to_be64((a << 1) | (b >> 63));
> + r->b = cpu_to_be64((b << 1) ^ _tt);
> +}
> +
> +/* needed by XTS */
> +static inline void gf128mul_x_ble(be128 *r, const be128 *x)
> +{
> + u64 a = le64_to_cpu(x->a);
> + u64 b = le64_to_cpu(x->b);
> +
> + /* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */
> + u64 _tt = gf128mul_mask_from_bit(b, 63) & 0x87;
> +
> + r->a = cpu_to_le64((a << 1) ^ _tt);
> + r->b = cpu_to_le64((b << 1) | (a >> 63));
> +}
>
> /* 4k table optimization */
>
> --
> 2.9.3
>
^ permalink raw reply
* Re: [PATCH v4] crypto: gf128mul - define gf128mul_x_* in gf128mul.h
From: Ondrej Mosnáček @ 2017-04-01 15:21 UTC (permalink / raw)
To: Herbert Xu
Cc: David S. Miller, linux-crypto, Jeffrey Walton, Milan Broz,
Ondrej Mosnacek, Eric Biggers
In-Reply-To: <CAAUqJDtE+ozFGodYAk=+LxQ9_Ot7mhCH3Asq2aP2RKRFZzz9Uw@mail.gmail.com>
Never mind, Gmail is confusing me... there is indeed "v4" in the subject :)
O.M.
2017-04-01 17:19 GMT+02:00 Ondrej Mosnáček <omosnacek@gmail.com>:
> Oops, sorry, wrong prefix...
>
> 2017-04-01 17:17 GMT+02:00 Ondrej Mosnacek <omosnacek@gmail.com>:
>> The gf128mul_x_ble function is currently defined in gf128mul.c, because
>> it depends on the gf128mul_table_be multiplication table.
>>
>> However, since the function is very small and only uses two values from
>> the table, it is better for it to be defined as inline function in
>> gf128mul.h. That way, the function can be inlined by the compiler for
>> better performance.
>>
>> For consistency, the other gf128mul_x_* functions are also moved to the
>> header file. In addition, the code is rewritten to be constant-time.
>>
>> After this change, the speed of the generic 'xts(aes)' implementation
>> increased from ~225 MiB/s to ~235 MiB/s (measured using 'cryptsetup
>> benchmark -c aes-xts-plain64' on an Intel system with CRYPTO_AES_X86_64
>> and CRYPTO_AES_NI_INTEL disabled).
>>
>> Signed-off-by: Ondrej Mosnacek <omosnacek@gmail.com>
>> Cc: Eric Biggers <ebiggers@google.com>
>> ---
>> v3 -> v4: a faster version of gf128mul_x_lle
>> v2 -> v3: constant-time implementation
>> v1 -> v2: move all _x_ functions to the header, not just gf128mul_x_ble
>>
>> crypto/gf128mul.c | 33 +---------------------------
>> include/crypto/gf128mul.h | 55 +++++++++++++++++++++++++++++++++++++++++++++--
>> 2 files changed, 54 insertions(+), 34 deletions(-)
>>
>> diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c
>> index 04facc0..dc01212 100644
>> --- a/crypto/gf128mul.c
>> +++ b/crypto/gf128mul.c
>> @@ -130,43 +130,12 @@ static const u16 gf128mul_table_le[256] = gf128mul_dat(xda_le);
>> static const u16 gf128mul_table_be[256] = gf128mul_dat(xda_be);
>>
>> /*
>> - * The following functions multiply a field element by x or by x^8 in
>> + * The following functions multiply a field element by x^8 in
>> * the polynomial field representation. They use 64-bit word operations
>> * to gain speed but compensate for machine endianness and hence work
>> * correctly on both styles of machine.
>> */
>>
>> -static void gf128mul_x_lle(be128 *r, const be128 *x)
>> -{
>> - u64 a = be64_to_cpu(x->a);
>> - u64 b = be64_to_cpu(x->b);
>> - u64 _tt = gf128mul_table_le[(b << 7) & 0xff];
>> -
>> - r->b = cpu_to_be64((b >> 1) | (a << 63));
>> - r->a = cpu_to_be64((a >> 1) ^ (_tt << 48));
>> -}
>> -
>> -static void gf128mul_x_bbe(be128 *r, const be128 *x)
>> -{
>> - u64 a = be64_to_cpu(x->a);
>> - u64 b = be64_to_cpu(x->b);
>> - u64 _tt = gf128mul_table_be[a >> 63];
>> -
>> - r->a = cpu_to_be64((a << 1) | (b >> 63));
>> - r->b = cpu_to_be64((b << 1) ^ _tt);
>> -}
>> -
>> -void gf128mul_x_ble(be128 *r, const be128 *x)
>> -{
>> - u64 a = le64_to_cpu(x->a);
>> - u64 b = le64_to_cpu(x->b);
>> - u64 _tt = gf128mul_table_be[b >> 63];
>> -
>> - r->a = cpu_to_le64((a << 1) ^ _tt);
>> - r->b = cpu_to_le64((b << 1) | (a >> 63));
>> -}
>> -EXPORT_SYMBOL(gf128mul_x_ble);
>> -
>> static void gf128mul_x8_lle(be128 *x)
>> {
>> u64 a = be64_to_cpu(x->a);
>> diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h
>> index 0bc9b5f..35ced9d 100644
>> --- a/include/crypto/gf128mul.h
>> +++ b/include/crypto/gf128mul.h
>> @@ -49,6 +49,7 @@
>> #ifndef _CRYPTO_GF128MUL_H
>> #define _CRYPTO_GF128MUL_H
>>
>> +#include <asm/byteorder.h>
>> #include <crypto/b128ops.h>
>> #include <linux/slab.h>
>>
>> @@ -163,8 +164,58 @@ void gf128mul_lle(be128 *a, const be128 *b);
>>
>> void gf128mul_bbe(be128 *a, const be128 *b);
>>
>> -/* multiply by x in ble format, needed by XTS */
>> -void gf128mul_x_ble(be128 *a, const be128 *b);
>> +/*
>> + * The following functions multiply a field element by x in
>> + * the polynomial field representation. They use 64-bit word operations
>> + * to gain speed but compensate for machine endianness and hence work
>> + * correctly on both styles of machine.
>> + *
>> + * They are defined here for performance.
>> + */
>> +
>> +static inline u64 gf128mul_mask_from_bit(u64 x, int which)
>> +{
>> + /* a constant-time version of 'x & ((u64)1 << which) ? (u64)-1 : 0' */
>> + return ((s64)(x << (63 - which)) >> 63);
>> +}
>> +
>> +static inline void gf128mul_x_lle(be128 *r, const be128 *x)
>> +{
>> + u64 a = be64_to_cpu(x->a);
>> + u64 b = be64_to_cpu(x->b);
>> +
>> + /* equivalent to gf128mul_table_le[(b << 7) & 0xff] << 48
>> + * (see crypto/gf128mul.c): */
>> + u64 _tt = gf128mul_mask_from_bit(b, 0) & ((u64)0xe1 << 56);
>> +
>> + r->b = cpu_to_be64((b >> 1) | (a << 63));
>> + r->a = cpu_to_be64((a >> 1) ^ _tt);
>> +}
>> +
>> +static inline void gf128mul_x_bbe(be128 *r, const be128 *x)
>> +{
>> + u64 a = be64_to_cpu(x->a);
>> + u64 b = be64_to_cpu(x->b);
>> +
>> + /* equivalent to gf128mul_table_be[a >> 63] (see crypto/gf128mul.c): */
>> + u64 _tt = gf128mul_mask_from_bit(a, 63) & 0x87;
>> +
>> + r->a = cpu_to_be64((a << 1) | (b >> 63));
>> + r->b = cpu_to_be64((b << 1) ^ _tt);
>> +}
>> +
>> +/* needed by XTS */
>> +static inline void gf128mul_x_ble(be128 *r, const be128 *x)
>> +{
>> + u64 a = le64_to_cpu(x->a);
>> + u64 b = le64_to_cpu(x->b);
>> +
>> + /* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */
>> + u64 _tt = gf128mul_mask_from_bit(b, 63) & 0x87;
>> +
>> + r->a = cpu_to_le64((a << 1) ^ _tt);
>> + r->b = cpu_to_le64((b << 1) | (a >> 63));
>> +}
>>
>> /* 4k table optimization */
>>
>> --
>> 2.9.3
>>
^ permalink raw reply
* Re: [PATCH 1/6] virtio: wrap find_vqs
From: Bjorn Andersson @ 2017-04-01 16:13 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: Dmitry Tarnyagin, kvm, David Airlie, linux-remoteproc, dri-devel,
virtualization, James E.J. Bottomley, Herbert Xu, linux-scsi,
John Fastabend, Arnd Bergmann, Amit Shah, Stefan Hajnoczi,
Martin K. Petersen, Greg Kroah-Hartman, linux-kernel,
linux-crypto, netdev, David S. Miller
In-Reply-To: <1490820507-8005-2-git-send-email-mst@redhat.com>
On Wed 29 Mar 13:48 PDT 2017, Michael S. Tsirkin wrote:
> We are going to add more parameters to find_vqs, let's wrap the call so
> we don't need to tweak all drivers every time.
>
> Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
Acked-by: Bjorn Andersson <bjorn.andersson@linaro.org>
Regards,
Bjorn
^ permalink raw reply
* [PATCH 0/5] crypto/nx: Enable NX 842 compression engine on Power9
From: Haren Myneni @ 2017-04-01 16:57 UTC (permalink / raw)
To: herbert, mpe, ddstreet
Cc: linuxppc-dev, linux-crypto, benh, mikey, suka, hbabu
[PATCH 0/5] crypto/nx: Enable NX 842 compression engine on Power9
P9 introduces Virtual Accelerator Switchboard (VAS) to communicate
with NX 842 engine. icswx function is used to access NX before.
On powerNV systems, NX-842 driver invokes VAS functions for
configuring RxFIFO (receive window) per each NX engine. VAS uses
this FIFO to communicate the request to NX. The kernel opens send
window which is used to transfer compression/decompression requests
to VAS. It maps the send window to the corresponding RxFIFO.
copy/paste instructions are used to pass the CRB to VAS.
This patch series adds P9 NX support for 842 compression engine.
First 3 patches reorganize the current code so that VAS function
can be added.
- nx842_powernv_function points to VAS function if VAS feature is
available. Otherwise icswx function is used.
- Move configure CRB code nx842_cfg_crb()
- In addition to freeing co-processor structs for initialization
failures and exit, both send and receive windows have to closed
for VAS.
The last 2 patches adds configuring and invoking VAS, and also
checking P9 NX specific errors that are provided in co-processor
status block (CSB) for failures.
Haren Myneni (5):
crypto/nx: Rename nx842_powernv_function as icswx function
crypto/nx: Create nx842_cfg_crb function
crypto/nx: Create nx842_delete_coproc function
crypto/nx: Add P9 NX support for 842 compression engine.
crypto/nx: Add P9 NX specific error codes for 842 engine
arch/powerpc/include/asm/icswx.h | 3 +
arch/powerpc/include/asm/vas.h | 2 +
drivers/crypto/nx/Kconfig | 1 +
drivers/crypto/nx/nx-842-powernv.c | 435 +++++++++++++++++++++++++++++++++----
drivers/crypto/nx/nx-842.h | 8 +
5 files changed, 407 insertions(+), 42 deletions(-)
--
1.8.3.1
^ permalink raw reply
* [PATCH 1/5] crypto/nx: Rename nx842_powernv_function as icswx function
From: Haren Myneni @ 2017-04-01 16:59 UTC (permalink / raw)
To: herbert, mpe, ddstreet
Cc: linuxppc-dev, linux-crypto, benh, mikey, suka, hbabu
[PATCH 1/5] crypto/nx: Rename nx842_powernv_function as icswx function
nx842_powernv_function is points to nx842_icswx_function and
will be point to VAS function which will be added later for
P9 NX support.
Signed-off-by: Haren Myneni <haren@us.ibm.com>
---
drivers/crypto/nx/nx-842-powernv.c | 24 +++++++++++++++++-------
1 file changed, 17 insertions(+), 7 deletions(-)
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
index 3abb045..125f1dc 100644
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ b/drivers/crypto/nx/nx-842-powernv.c
@@ -47,14 +47,22 @@ struct nx842_workmem {
struct nx842_coproc {
unsigned int chip_id;
- unsigned int ct;
- unsigned int ci;
+ union {
+ struct {
+ unsigned int ct;
+ unsigned int ci;
+ } icswx;
+ };
struct list_head list;
};
/* no cpu hotplug on powernv, so this list never changes after init */
static LIST_HEAD(nx842_coprocs);
-static unsigned int nx842_ct;
+static unsigned int nx842_ct; /* use with icswx function */
+
+static int (*nx842_powernv_function)(const unsigned char *in,
+ unsigned int inlen, unsigned char *out,
+ unsigned int *outlenp, void *workmem, int fc);
/**
* setup_indirect_dde - Setup an indirect DDE
@@ -355,7 +363,7 @@ static int wait_for_csb(struct nx842_workmem *wmem,
}
/**
- * nx842_powernv_function - compress/decompress data using the 842 algorithm
+ * nx842_icswx_function - compress/decompress data using the 842 algorithm
*
* (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
* This compresses or decompresses the provided input buffer into the provided
@@ -385,7 +393,7 @@ static int wait_for_csb(struct nx842_workmem *wmem,
* -ETIMEDOUT hardware did not complete operation in reasonable time
* -EINTR operation was aborted
*/
-static int nx842_powernv_function(const unsigned char *in, unsigned int inlen,
+static int nx842_icswx_function(const unsigned char *in, unsigned int inlen,
unsigned char *out, unsigned int *outlenp,
void *workmem, int fc)
{
@@ -554,8 +562,8 @@ static int __init nx842_powernv_probe(struct device_node *dn)
return -ENOMEM;
coproc->chip_id = chip_id;
- coproc->ct = ct;
- coproc->ci = ci;
+ coproc->icswx.ct = ct;
+ coproc->icswx.ci = ci;
INIT_LIST_HEAD(&coproc->list);
list_add(&coproc->list, &nx842_coprocs);
@@ -625,6 +633,8 @@ static __init int nx842_powernv_init(void)
if (!nx842_ct)
return -ENODEV;
+ nx842_powernv_function = nx842_icswx_function;
+
ret = crypto_register_alg(&nx842_powernv_alg);
if (ret) {
struct nx842_coproc *coproc, *n;
--
1.8.3.1
^ permalink raw reply related
* [PATCH 2/5] crypto/nx: Create nx842_cfg_crb function
From: Haren Myneni @ 2017-04-01 17:00 UTC (permalink / raw)
To: herbert, mpe, ddstreet
Cc: linuxppc-dev, linux-crypto, benh, mikey, suka, hbabu
[PATCH 2/5] crypto/nx: Create nx842_cfg_crb function
Configure CRB is moved to nx842_cfg_crb() so that it can be
used for icswx function and VAS function which will be added
later.
Signed-off-by: Haren Myneni <haren@us.ibm.com>
---
drivers/crypto/nx/nx-842-powernv.c | 57 +++++++++++++++++++++++++-------------
1 file changed, 38 insertions(+), 19 deletions(-)
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
index 125f1dc..4cd6a6f 100644
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ b/drivers/crypto/nx/nx-842-powernv.c
@@ -362,6 +362,40 @@ static int wait_for_csb(struct nx842_workmem *wmem,
return 0;
}
+static int nx842_cfg_crb(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int outlen,
+ struct nx842_workmem *wmem)
+{
+ struct coprocessor_request_block *crb;
+ struct coprocessor_status_block *csb;
+ u64 csb_addr;
+ int ret;
+
+ crb = &wmem->crb;
+ csb = &crb->csb;
+
+ /* Clear any previous values */
+ memset(crb, 0, sizeof(*crb));
+
+ /* set up DDLs */
+ ret = setup_ddl(&crb->source, wmem->ddl_in,
+ (unsigned char *)in, inlen, true);
+ if (ret)
+ return ret;
+
+ ret = setup_ddl(&crb->target, wmem->ddl_out,
+ out, outlen, false);
+ if (ret)
+ return ret;
+
+ /* set up CRB's CSB addr */
+ csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS;
+ csb_addr |= CRB_CSB_AT; /* Addrs are phys */
+ crb->csb_addr = cpu_to_be64(csb_addr);
+
+ return 0;
+}
+
/**
* nx842_icswx_function - compress/decompress data using the 842 algorithm
*
@@ -401,7 +435,6 @@ static int nx842_icswx_function(const unsigned char *in, unsigned int inlen,
struct coprocessor_status_block *csb;
struct nx842_workmem *wmem;
int ret;
- u64 csb_addr;
u32 ccw;
unsigned int outlen = *outlenp;
@@ -415,33 +448,19 @@ static int nx842_icswx_function(const unsigned char *in, unsigned int inlen,
return -ENODEV;
}
- crb = &wmem->crb;
- csb = &crb->csb;
-
- /* Clear any previous values */
- memset(crb, 0, sizeof(*crb));
-
- /* set up DDLs */
- ret = setup_ddl(&crb->source, wmem->ddl_in,
- (unsigned char *)in, inlen, true);
- if (ret)
- return ret;
- ret = setup_ddl(&crb->target, wmem->ddl_out,
- out, outlen, false);
+ ret = nx842_cfg_crb(in, inlen, out, outlen, wmem);
if (ret)
return ret;
+ crb = &wmem->crb;
+ csb = &crb->csb;
+
/* set up CCW */
ccw = 0;
ccw = SET_FIELD(CCW_CT, ccw, nx842_ct);
ccw = SET_FIELD(CCW_CI_842, ccw, 0); /* use 0 for hw auto-selection */
ccw = SET_FIELD(CCW_FC_842, ccw, fc);
- /* set up CRB's CSB addr */
- csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS;
- csb_addr |= CRB_CSB_AT; /* Addrs are phys */
- crb->csb_addr = cpu_to_be64(csb_addr);
-
wmem->start = ktime_get();
/* do ICSWX */
--
1.8.3.1
^ permalink raw reply related
* [PATCH 3/5] crypto/nx: Create nx842_delete_coproc function
From: Haren Myneni @ 2017-04-01 17:00 UTC (permalink / raw)
To: herbert, mpe, ddstreet
Cc: linuxppc-dev, linux-crypto, benh, mikey, suka, hbabu
[PATCH 3/5] crypto/nx: Create nx842_delete_coproc function
Move deleting coprocessor info upon exit or failure to
nx842_delete_coproc().
Signed-off-by: Haren Myneni <haren@us.ibm.com>
---
drivers/crypto/nx/nx-842-powernv.c | 25 ++++++++++++-------------
1 file changed, 12 insertions(+), 13 deletions(-)
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
index 4cd6a6f..8737e90 100644
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ b/drivers/crypto/nx/nx-842-powernv.c
@@ -597,6 +597,16 @@ static int __init nx842_powernv_probe(struct device_node *dn)
return 0;
}
+static void nx842_delete_coproc(void)
+{
+ struct nx842_coproc *coproc, *n;
+
+ list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+ list_del(&coproc->list);
+ kfree(coproc);
+ }
+}
+
static struct nx842_constraints nx842_powernv_constraints = {
.alignment = DDE_BUFFER_ALIGN,
.multiple = DDE_BUFFER_LAST_MULT,
@@ -656,13 +666,7 @@ static __init int nx842_powernv_init(void)
ret = crypto_register_alg(&nx842_powernv_alg);
if (ret) {
- struct nx842_coproc *coproc, *n;
-
- list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
- list_del(&coproc->list);
- kfree(coproc);
- }
-
+ nx842_delete_coproc();
return ret;
}
@@ -672,13 +676,8 @@ static __init int nx842_powernv_init(void)
static void __exit nx842_powernv_exit(void)
{
- struct nx842_coproc *coproc, *n;
-
crypto_unregister_alg(&nx842_powernv_alg);
- list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
- list_del(&coproc->list);
- kfree(coproc);
- }
+ nx842_delete_coproc();
}
module_exit(nx842_powernv_exit);
--
1.8.3.1
^ permalink raw reply related
* [PATCH 4/5] crypto/nx: Add P9 NX support for 842 compression engine.
From: Haren Myneni @ 2017-04-01 17:01 UTC (permalink / raw)
To: herbert, mpe, ddstreet
Cc: linuxppc-dev, linux-crypto, benh, mikey, suka, hbabu
[PATCH 4/5] crypto/nx: Add P9 NX support for 842 compression engine.
This patch adds P9 NX support for 842 compression engine. Virtual
Accelerator Switchboard (VAS) is used to access 842 engine on P9.
For each NX engine per chip, setup receive window using
vas_rx_win_open() which configures RxFIFo with FIFO address, lpid,
pid and tid values. This unique (lpid, pid, tid) combination will
be used to identify the target engine. Then open send window with
vas_tx_win_open() which is used to send NX requests to VAS. NX
provides high and normal priority FIFOs. For compression /
decompression requests, we use only hight priority FIFOs in kernel.
Each NX request will be communicated to VAS using copy/paste
instructions with vas_copy_crb() / vas_paste_crb() functions.
Signed-off-by: Haren Myneni <haren@us.ibm.com>
---
arch/powerpc/include/asm/vas.h | 2 +
drivers/crypto/nx/Kconfig | 1 +
drivers/crypto/nx/nx-842-powernv.c | 315 ++++++++++++++++++++++++++++++++++++-
3 files changed, 313 insertions(+), 5 deletions(-)
diff --git a/arch/powerpc/include/asm/vas.h b/arch/powerpc/include/asm/vas.h
index 4e5a470..7315621 100644
--- a/arch/powerpc/include/asm/vas.h
+++ b/arch/powerpc/include/asm/vas.h
@@ -19,6 +19,8 @@
#define VAS_RX_FIFO_SIZE_MIN (1 << 10) /* 1KB */
#define VAS_RX_FIFO_SIZE_MAX (8 << 20) /* 8MB */
+#define is_vas_available() (cpu_has_feature(CPU_FTR_ARCH_300))
+
/*
* Co-processor Engine type.
*/
diff --git a/drivers/crypto/nx/Kconfig b/drivers/crypto/nx/Kconfig
index ad7552a..4ad7fdb 100644
--- a/drivers/crypto/nx/Kconfig
+++ b/drivers/crypto/nx/Kconfig
@@ -38,6 +38,7 @@ config CRYPTO_DEV_NX_COMPRESS_PSERIES
config CRYPTO_DEV_NX_COMPRESS_POWERNV
tristate "Compression acceleration support on PowerNV platform"
depends on PPC_POWERNV
+ select VAS
default y
help
Support for PowerPC Nest (NX) compression acceleration. This
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
index 8737e90..66efd39 100644
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ b/drivers/crypto/nx/nx-842-powernv.c
@@ -23,6 +23,7 @@
#include <asm/prom.h>
#include <asm/icswx.h>
#include <asm/vas.h>
+#include <asm/reg.h>
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
@@ -52,10 +53,20 @@ struct nx842_coproc {
unsigned int ct;
unsigned int ci;
} icswx;
+ struct {
+ struct vas_window *txwin;
+ struct vas_window *rxwin;
+ } vas;
};
struct list_head list;
};
+/*
+ * Send the request to NX engine on the chip for the corresponding CPU
+ * where the process is executing. Use with VAS function.
+ */
+static DEFINE_PER_CPU(struct vas_window *, nx842_txwin);
+
/* no cpu hotplug on powernv, so this list never changes after init */
static LIST_HEAD(nx842_coprocs);
static unsigned int nx842_ct; /* use with icswx function */
@@ -499,6 +510,103 @@ static int nx842_icswx_function(const unsigned char *in, unsigned int inlen,
}
/**
+ * nx842_vas_function - compress/decompress data using the 842 algorithm
+ *
+ * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
+ * This compresses or decompresses the provided input buffer into the provided
+ * output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * output data. If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * The @workmem buffer should only be used by one function call at a time.
+ *
+ * @in: input buffer pointer
+ * @inlen: input buffer size
+ * @out: output buffer pointer
+ * @outlenp: output buffer size pointer
+ * @workmem: working memory buffer pointer, size determined by
+ * nx842_powernv_driver.workmem_size
+ * @fc: function code, see CCW Function Codes in nx-842.h
+ *
+ * Returns:
+ * 0 Success, output of length @outlenp stored in the buffer
+ * at @out
+ * -ENODEV Hardware unavailable
+ * -ENOSPC Output buffer is to small
+ * -EMSGSIZE Input buffer too large
+ * -EINVAL buffer constraints do not fix nx842_constraints
+ * -EPROTO hardware error during operation
+ * -ETIMEDOUT hardware did not complete operation in reasonable time
+ * -EINTR operation was aborted
+ */
+static int nx842_vas_function(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlenp,
+ void *workmem, int fc)
+{
+ struct coprocessor_request_block *crb;
+ struct coprocessor_status_block *csb;
+ struct nx842_workmem *wmem;
+ struct vas_window *txwin;
+ int ret;
+ u32 ccw;
+ unsigned int outlen = *outlenp;
+
+ wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
+
+ *outlenp = 0;
+
+ crb = &wmem->crb;
+ csb = &crb->csb;
+
+ ret = nx842_cfg_crb(in, inlen, out, outlen, wmem);
+ if (ret)
+ return ret;
+
+ ccw = 0;
+ ccw = SET_FIELD(CCW_FC_842, ccw, fc);
+ crb->ccw = cpu_to_be32(ccw);
+
+ /*
+ * Send request to the NX engine on the chip that corresponds to
+ * the current CPU.
+ */
+ txwin = per_cpu(nx842_txwin, smp_processor_id());
+ /* shoudn't happen, we don't load without a coproc */
+ if (!txwin) {
+ pr_err_ratelimited("NX-842 coprocessor is not available");
+ return -ENODEV;
+ }
+
+ wmem->start = ktime_get();
+
+ /*
+ * VAS copy CRB into L2 cache. Refer <asm/vas.h>.
+ * @crb, @offset and @first (must be true)
+ */
+ vas_copy_crb(crb, 0, 1);
+
+ /*
+ * VAS paste previously copied CRB to NX.
+ * @txwin, @offset, @last (must be true) and @re is expected/assumed
+ * to be true for NX windows.
+ */
+ ret = vas_paste_crb(txwin, 0, 1, 1);
+
+ if (ret) {
+ pr_err_ratelimited("NX-842: VAS copy/paste failed\n");
+ return ret;
+ }
+
+ ret = wait_for_csb(wmem, csb);
+ if (!ret)
+ *outlenp = be32_to_cpu(csb->count);
+
+ return ret;
+}
+
+/**
* nx842_powernv_compress - Compress data using the 842 algorithm
*
* Compression provided by the NX842 coprocessor on IBM PowerNV systems.
@@ -554,6 +662,164 @@ static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen,
wmem, CCW_FC_842_DECOMP_CRC);
}
+
+static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
+ int vasid, int ct)
+{
+ struct vas_window *rxwin, *txwin = NULL;
+ struct vas_rx_win_attr rxattr;
+ struct vas_tx_win_attr txattr;
+ struct nx842_coproc *coproc;
+ u32 lpid, pid, tid;
+ u64 rx_fifo;
+ int ret;
+#define RX_FIFO_SIZE 0x8000
+
+ if (of_property_read_u64(dn, "rx-fifo-address", (void *)&rx_fifo)) {
+ pr_err("ibm,nx-842: Missing rx-fifo-address property\n");
+ return -EINVAL;
+ }
+
+ if (of_property_read_u32(dn, "lpid", &lpid)) {
+ pr_err("ibm,nx-842: Missing lpid property\n");
+ return -EINVAL;
+ }
+
+ if (of_property_read_u32(dn, "pid", &pid)) {
+ pr_err("ibm,nx-842: Missing pid property\n");
+ return -EINVAL;
+ }
+
+ if (of_property_read_u32(dn, "tid", &tid)) {
+ pr_err("ibm,nx-842: Missing tid property\n");
+ return -EINVAL;
+ }
+
+ vas_init_rx_win_attr(&rxattr, ct);
+ rxattr.rx_fifo = (void *)rx_fifo;
+ rxattr.rx_fifo_size = RX_FIFO_SIZE;
+ rxattr.lnotify_lpid = lpid;
+ rxattr.lnotify_pid = pid;
+ rxattr.lnotify_tid = tid;
+ rxattr.wcreds_max = 64;
+
+ /*
+ * Open a VAS receice window which is used to configure RxFIFO
+ * for NX.
+ */
+ rxwin = vas_rx_win_open(vasid, ct, &rxattr);
+ if (IS_ERR(rxwin)) {
+ pr_err("ibm,nx-842: setting RxFIFO with VAS failed: %ld\n",
+ PTR_ERR(rxwin));
+ return PTR_ERR(rxwin);
+ }
+
+ /*
+ * Kernel requests will be high priority. So open send
+ * windows only for high priority RxFIFO entries.
+ */
+ if (ct == VAS_COP_TYPE_842_HIPRI) {
+ vas_init_tx_win_attr(&txattr, ct);
+ txattr.lpid = 0; /* lpid is 0 for kernel requests */
+ txattr.pid = mfspr(SPRN_PID);
+
+ /*
+ * Open a VAS send window which is used to send request to NX.
+ */
+ txwin = vas_tx_win_open(vasid, ct, &txattr);
+ if (IS_ERR(txwin)) {
+ pr_err("ibm,nx-842: Can not open TX window: %ld\n",
+ PTR_ERR(txwin));
+ ret = PTR_ERR(txwin);
+ goto err_out;
+ }
+ }
+
+ coproc = kmalloc(sizeof(*coproc), GFP_KERNEL);
+ if (!coproc) {
+ ret = -ENOMEM;
+ goto err_out;
+ }
+
+ coproc->chip_id = chip_id;
+ coproc->vas.rxwin = rxwin;
+ coproc->vas.txwin = txwin;
+
+ INIT_LIST_HEAD(&coproc->list);
+ list_add(&coproc->list, &nx842_coprocs);
+
+ return 0;
+
+err_out:
+ if (txwin)
+ vas_win_close(txwin);
+
+ vas_win_close(rxwin);
+
+ return ret;
+}
+
+
+static int __init nx842_powernv_probe_vas(struct device_node *dn)
+{
+ struct device_node *nxdn, *np;
+ int chip_id, vasid, rc;
+
+ chip_id = of_get_ibm_chip_id(dn);
+ if (chip_id < 0) {
+ pr_err("ibm,chip-id missing\n");
+ return -EINVAL;
+ }
+
+ np = of_find_node_by_name(dn, "vas");
+ if (!np) {
+ pr_err("ibm,xscom: Missing VAS device node\n");
+ return -EINVAL;
+ }
+
+ if (of_property_read_u32(np, "vas-id", &vasid)) {
+ pr_err("ibm,vas: Missing vas-id device property\n");
+ of_node_put(np);
+ return -EINVAL;
+ }
+
+ of_node_put(np);
+
+ nxdn = of_find_compatible_node(dn, NULL, "ibm,power-nx");
+ if (!nxdn) {
+ pr_err("ibm,xscom: Missing NX device node\n");
+ return -EINVAL;
+ }
+
+ np = of_find_node_by_name(nxdn, "ibm,nx-842-high");
+ if (!np) {
+ pr_err("ibm,nx-842-high device node is missing\n");
+ rc = -EINVAL;
+ goto out_nd_put;
+ }
+
+ rc = vas_cfg_coproc_info(np, chip_id, vasid, VAS_COP_TYPE_842_HIPRI);
+ of_node_put(np);
+ if (rc)
+ goto out_nd_put;
+
+ np = of_find_node_by_name(nxdn, "ibm,nx-842-normal");
+ if (!np) {
+ pr_err("ibm,nx-842-normal device node is missing\n");
+ rc = -EINVAL;
+ goto out_nd_put;
+ }
+
+ rc = vas_cfg_coproc_info(np, chip_id, vasid, VAS_COP_TYPE_842);
+ of_node_put(np);
+ if (!rc)
+ return 0;
+
+out_nd_put:
+ of_node_put(nxdn);
+ return rc;
+}
+
static int __init nx842_powernv_probe(struct device_node *dn)
{
struct nx842_coproc *coproc;
@@ -602,11 +868,42 @@ static void nx842_delete_coproc(void)
struct nx842_coproc *coproc, *n;
list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+ if (is_vas_available()) {
+ vas_win_close(coproc->vas.rxwin);
+ /*
+ * txwin opened only for high priority RxFIFOs
+ */
+ if (coproc->vas.txwin)
+ vas_win_close(coproc->vas.txwin);
+ }
list_del(&coproc->list);
kfree(coproc);
}
}
+/*
+ * Identify chip ID for each CPU and save send window address in percpu
+ * nx842_txwin for the corresponding NX engine.
+ * nx842_txwin is used later to send the NX request with the txwin for
+ * the CPU where the request is running.
+ */
+static void nx842_set_per_cpu_txwin(void)
+{
+ unsigned int i, chip_id;
+ struct nx842_coproc *coproc, *n;
+
+ for_each_possible_cpu(i) {
+ chip_id = cpu_to_chip_id(i);
+ list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+ if ((coproc->chip_id == chip_id) &&
+ coproc->vas.txwin) {
+ per_cpu(nx842_txwin, i) = coproc->vas.txwin;
+ break;
+ }
+ }
+ }
+}
+
static struct nx842_constraints nx842_powernv_constraints = {
.alignment = DDE_BUFFER_ALIGN,
.multiple = DDE_BUFFER_LAST_MULT,
@@ -656,13 +953,21 @@ static __init int nx842_powernv_init(void)
BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT);
BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
- for_each_compatible_node(dn, NULL, "ibm,power-nx")
- nx842_powernv_probe(dn);
+ if (is_vas_available()) {
+ for_each_compatible_node(dn, NULL, "ibm,xscom")
+ nx842_powernv_probe_vas(dn);
- if (!nx842_ct)
- return -ENODEV;
+ nx842_set_per_cpu_txwin();
+ nx842_powernv_function = nx842_vas_function;
+ } else {
+ for_each_compatible_node(dn, NULL, "ibm,power-nx")
+ nx842_powernv_probe(dn);
- nx842_powernv_function = nx842_icswx_function;
+ nx842_powernv_function = nx842_icswx_function;
+ }
+
+ if (list_empty(&nx842_coprocs))
+ return -ENODEV;
ret = crypto_register_alg(&nx842_powernv_alg);
if (ret) {
--
1.8.3.1
^ permalink raw reply related
* [PATCH 5/5] crypto/nx: Add P9 NX specific error codes for 842 engine
From: Haren Myneni @ 2017-04-01 17:02 UTC (permalink / raw)
To: herbert, mpe, ddstreet
Cc: linuxppc-dev, linux-crypto, benh, mikey, suka, hbabu
[PATCH 5/5] crypto/nx: Add P9 NX specific error codes for 842 engine
This patch adds changes for checking P9 specific 842 engine
error codes. These errros are reported in co-processor status
block (CSB) for failures.
Signed-off-by: Haren Myneni <haren@us.ibm.com>
---
arch/powerpc/include/asm/icswx.h | 3 +++
drivers/crypto/nx/nx-842-powernv.c | 18 ++++++++++++++++++
drivers/crypto/nx/nx-842.h | 8 ++++++++
3 files changed, 29 insertions(+)
diff --git a/arch/powerpc/include/asm/icswx.h b/arch/powerpc/include/asm/icswx.h
index 27e588f..6a2c875 100644
--- a/arch/powerpc/include/asm/icswx.h
+++ b/arch/powerpc/include/asm/icswx.h
@@ -69,7 +69,10 @@ struct coprocessor_completion_block {
#define CSB_CC_WR_PROTECTION (16)
#define CSB_CC_UNKNOWN_CODE (17)
#define CSB_CC_ABORT (18)
+#define CSB_CC_EXCEED_BYTE_COUNT (19) /* P9 or later */
#define CSB_CC_TRANSPORT (20)
+#define CSB_CC_INVALID_CRB (21) /* P9 or later */
+#define CSB_CC_INVALID_DDE (30) /* P9 or later */
#define CSB_CC_SEGMENTED_DDL (31)
#define CSB_CC_PROGRESS_POINT (32)
#define CSB_CC_DDE_OVERFLOW (33)
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
index 66efd39..719d9f1 100644
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ b/drivers/crypto/nx/nx-842-powernv.c
@@ -258,6 +258,13 @@ static int wait_for_csb(struct nx842_workmem *wmem,
case CSB_CC_TEMPL_OVERFLOW:
CSB_ERR(csb, "Compressed data template shows data past end");
return -EINVAL;
+ case CSB_CC_EXCEED_BYTE_COUNT: /* P9 or later */
+ /*
+ * DDE byte count exceeds the limit specified in Maximum
+ * byte count register.
+ */
+ CSB_ERR(csb, "DDE byte count exceeds the limit");
+ return -EINVAL;
/* these should not happen */
case CSB_CC_INVALID_ALIGN:
@@ -299,9 +306,17 @@ static int wait_for_csb(struct nx842_workmem *wmem,
CSB_ERR(csb, "Too many DDEs in DDL");
return -EINVAL;
case CSB_CC_TRANSPORT:
+ case CSB_CC_INVALID_CRB: /* P9 or later */
/* shouldn't happen, we setup CRB correctly */
CSB_ERR(csb, "Invalid CRB");
return -EINVAL;
+ case CSB_CC_INVALID_DDE: /* P9 or later */
+ /*
+ * shouldn't happen, setup_direct/indirect_dde creates
+ * DDE right
+ */
+ CSB_ERR(csb, "Invalid DDE");
+ return -EINVAL;
case CSB_CC_SEGMENTED_DDL:
/* shouldn't happen, setup_ddl creates DDL right */
CSB_ERR(csb, "Segmented DDL error");
@@ -345,6 +360,9 @@ static int wait_for_csb(struct nx842_workmem *wmem,
case CSB_CC_HW:
CSB_ERR(csb, "Correctable hardware error");
return -EPROTO;
+ case CSB_CC_HW_EXPIRED_TIMER: /* P9 or later */
+ CSB_ERR(csb, "Job did not finish within allowed time");
+ return -EPROTO;
default:
CSB_ERR(csb, "Invalid CC %d", csb->cc);
diff --git a/drivers/crypto/nx/nx-842.h b/drivers/crypto/nx/nx-842.h
index 30929bd..ed06112 100644
--- a/drivers/crypto/nx/nx-842.h
+++ b/drivers/crypto/nx/nx-842.h
@@ -76,9 +76,17 @@
#define CSB_CC_DECRYPT_OVERFLOW (64)
/* asym crypt codes */
#define CSB_CC_MINV_OVERFLOW (128)
+/*
+ * HW error - Job did not finish in the maximum time allowed.
+ * Jon terminated.
+ */
+#define CSB_CC_HW_EXPIRED_TIMER (224)
/* These are reserved for hypervisor use */
#define CSB_CC_HYP_RESERVE_START (240)
#define CSB_CC_HYP_RESERVE_END (253)
+#define CSB_CC_HYP_RESERVE_P9_END (251)
+/* No valid interrupt server (P9 or later). */
+#define CSB_CC_HYP_RESERVE_NO_INTR_SERVER (252)
#define CSB_CC_HYP_NO_HW (254)
#define CSB_CC_HYP_HANG_ABORTED (255)
--
1.8.3.1
^ permalink raw reply related
* [RESEND PATCH 0/5] Enable NX 842 compression engine on Power9
From: Haren Myneni @ 2017-04-01 17:10 UTC (permalink / raw)
To: herbert, mpe, ddstreet
Cc: linuxppc-dev, linux-crypto, benh, mikey, suka, hbabu
Sorry for reposting. Missed to specify dependency - VAS kernel changes.
[PATCH 0/5] Enable NX 842 compression engine on Power9
P9 introduces Virtual Accelerator Switchboard (VAS) to communicate
with NX 842 engine. icswx function is used to access NX before.
On powerNV systems, NX-842 driver invokes VAS functions for
configuring RxFIFO (receive window) per each NX engine. VAS uses
this FIFO to communicate the request to NX. The kernel opens send
window which is used to transfer compression/decompression requests
to VAS. It maps the send window to the corresponding RxFIFO.
copy/paste instructions are used to pass the CRB to VAS.
This patch series adds P9 NX support for 842 compression engine.
First 3 patches reorganize the current code so that VAS function
can be added.
- nx842_powernv_function points to VAS function if VAS feature is
available. Otherwise icswx function is used.
- Move configure CRB code nx842_cfg_crb()
- In addition to freeing co-processor structs for initialization
failures and exit, both send and receive windows have to closed
for VAS.
The last 2 patches adds configuring and invoking VAS, and also
checking P9 NX specific errors that are provided in co-processor
status block (CSB) for failures.
Patches have been tested in P9 Simics environment using VAS changes.
This patchset depends on VAS kernel changes:
https://lists.ozlabs.org/pipermail/linuxppc-dev/2017-March/156078.html
Thanks to Sukadev Bhattiprolu for his review, input and testing with
VAS changes.
Haren Myneni (5):
crypto/nx: Rename nx842_powernv_function as icswx function
crypto/nx: Create nx842_cfg_crb function
crypto/nx: Create nx842_delete_coproc function
crypto/nx: Add P9 NX support for 842 compression engine.
crypto/nx: Add P9 NX specific error codes for 842 engine
arch/powerpc/include/asm/icswx.h | 3 +
arch/powerpc/include/asm/vas.h | 2 +
drivers/crypto/nx/Kconfig | 1 +
drivers/crypto/nx/nx-842-powernv.c | 435 +++++++++++++++++++++++++++++++++----
drivers/crypto/nx/nx-842.h | 8 +
5 files changed, 407 insertions(+), 42 deletions(-)
--
1.8.3.1
^ permalink raw reply
* [PATCH] crypto/nx: Update MAINTAINERS entry for 842 compression
From: Haren Myneni @ 2017-04-01 17:25 UTC (permalink / raw)
To: herbert, ddstreet; +Cc: linux-crypto, linux-kernel
[PATCH] crypto/nx: Update MAINTAINERS entry for 842 compression
Signed-off-by: Haren Myneni <haren@us.ibm.com>
---
MAINTAINERS | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/MAINTAINERS b/MAINTAINERS
index c265a5f..4cfd225 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -6211,7 +6211,7 @@ F: drivers/crypto/nx/nx_csbcpb.h
F: drivers/crypto/nx/nx_debugfs.h
IBM Power 842 compression accelerator
-M: Dan Streetman <ddstreet@ieee.org>
+M: Haren Myneni <haren@us.ibm.com>
S: Supported
F: drivers/crypto/nx/Makefile
F: drivers/crypto/nx/Kconfig
--
1.8.3.1
^ permalink raw reply related
* Re: [PATCH] crypto: AF_ALG: handle 0 lengths in af_alg_make_sg
From: Stephan Müller @ 2017-04-01 17:46 UTC (permalink / raw)
To: herbert; +Cc: linux-crypto
In-Reply-To: <2380853.MJYDG7HQLD@positron.chronox.de>
Am Samstag, 1. April 2017, 17:04:28 CEST schrieb Stephan Müller:
Hi Herbert,
> Hi Herbert,
>
> If you concur with the patch, I think it should go to 4.11 as well as
> to stable.
After checking this issue again, I see that it is not triggerable in the
current code as the different af_alg users make sure that this function is not
called with 0.
I only triggered this issue during experimenting with the algif_skcipher and
algif_aead revamp as requested by you. During those experiments, I invoked
af_alg_make_sg with a len = 0.
Thus, this patch is not applicable for stable and 4.11.
Yet, I would suggest to consider this patch as a safeguard for any potential
programming errors.
Ciao
Stephan
^ permalink raw reply
* Re: [PATCH] crypto/nx: Update MAINTAINERS entry for 842 compression
From: Dan Streetman @ 2017-04-01 18:36 UTC (permalink / raw)
To: Haren Myneni; +Cc: Herbert Xu, Linux Crypto Mailing List, linux-kernel
In-Reply-To: <1491067540.29552.39.camel@hbabu-laptop>
On Sat, Apr 1, 2017 at 1:25 PM, Haren Myneni <haren@linux.vnet.ibm.com> wrote:
> [PATCH] crypto/nx: Update MAINTAINERS entry for 842 compression
>
> Signed-off-by: Haren Myneni <haren@us.ibm.com>
Acked-by: Dan Streetman <ddstreet@ieee.org>
>
> ---
> MAINTAINERS | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/MAINTAINERS b/MAINTAINERS
> index c265a5f..4cfd225 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -6211,7 +6211,7 @@ F: drivers/crypto/nx/nx_csbcpb.h
> F: drivers/crypto/nx/nx_debugfs.h
>
> IBM Power 842 compression accelerator
> -M: Dan Streetman <ddstreet@ieee.org>
> +M: Haren Myneni <haren@us.ibm.com>
> S: Supported
> F: drivers/crypto/nx/Makefile
> F: drivers/crypto/nx/Kconfig
> --
> 1.8.3.1
>
>
>
^ permalink raw reply
* [PATCH v6 2/2] crypto: aead AF_ALG - overhaul memory management
From: Stephan Müller @ 2017-04-01 20:53 UTC (permalink / raw)
To: herbert; +Cc: linux-crypto
In-Reply-To: <2473716.zZb2vDUueJ@positron.chronox.de>
The updated memory management is described in the top part of the code.
As one benefit of the changed memory management, the AIO and synchronous
operation is now implemented in one common function. The AF_ALG
operation uses the async kernel crypto API interface for each cipher
operation. Thus, the only difference between the AIO and sync operation
types visible from user space is:
1. the callback function to be invoked when the asynchronous operation
is completed
2. whether to wait for the completion of the kernel crypto API operation
or not
The change includes the overhaul of the TX and RX SGL handling. The TX
SGL holding the data sent from user space to the kernel is now dynamic
similar to algif_skcipher. This dynamic nature allows a continuous
operation of a thread sending data and a second thread receiving the
data. These threads do not need to synchronize as the kernel processes
as much data from the TX SGL to fill the RX SGL.
The caller reading the data from the kernel defines the amount of data
to be processed. Considering that the interface covers AEAD
authenticating ciphers, the reader must provide the buffer in the
correct size. Thus the reader defines the encryption size.
Signed-off-by: Stephan Mueller <smueller@chronox.de>
---
crypto/algif_aead.c | 762 ++++++++++++++++++++++++++++++----------------------
1 file changed, 441 insertions(+), 321 deletions(-)
diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c
index 5a80537..be49c32 100644
--- a/crypto/algif_aead.c
+++ b/crypto/algif_aead.c
@@ -5,12 +5,26 @@
*
* This file provides the user-space API for AEAD ciphers.
*
- * This file is derived from algif_skcipher.c.
- *
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
+ *
+ * The following concept of the memory management is used:
+ *
+ * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is
+ * filled by user space with the data submitted via sendpage/sendmsg. Filling
+ * up the TX SGL does not cause a crypto operation -- the data will only be
+ * tracked by the kernel. Upon receipt of one recvmsg call, the caller must
+ * provide a buffer which is tracked with the RX SGL.
+ *
+ * During the processing of the recvmsg operation, the cipher request is
+ * allocated and prepared. As part of the recvmsg operation, the processed
+ * TX buffers are extracted from the TX SGL into a separate SGL.
+ *
+ * After the completion of the crypto operation, the RX SGL and the cipher
+ * request is released. The extracted TX SGL parts are released together with
+ * the RX SGL release.
*/
#include <crypto/internal/aead.h>
@@ -25,45 +39,59 @@
#include <linux/net.h>
#include <net/sock.h>
-struct aead_sg_list {
- unsigned int cur;
- struct scatterlist sg[ALG_MAX_PAGES];
+struct aead_tsgl {
+ struct list_head list;
+ unsigned int cur; /* Last processed SG entry */
+ struct scatterlist sg[0]; /* Array of SGs forming the SGL */
};
-struct aead_async_rsgl {
+struct aead_rsgl {
struct af_alg_sgl sgl;
struct list_head list;
+ size_t sg_num_bytes; /* Bytes of data in that SGL */
};
struct aead_async_req {
- struct scatterlist *tsgl;
- struct aead_async_rsgl first_rsgl;
- struct list_head list;
struct kiocb *iocb;
- unsigned int tsgls;
- char iv[];
+ struct sock *sk;
+
+ struct aead_rsgl first_rsgl; /* First RX SG */
+ struct list_head rsgl_list; /* Track RX SGs */
+
+ struct scatterlist *tsgl; /* priv. TX SGL of buffers to process */
+ unsigned int tsgl_entries; /* number of entries in priv. TX SGL */
+
+ unsigned int outlen; /* Filled output buf length */
+
+ unsigned int areqlen; /* Length of this data struct */
+ struct aead_request aead_req; /* req ctx trails this struct */
};
struct aead_ctx {
- struct aead_sg_list tsgl;
- struct aead_async_rsgl first_rsgl;
- struct list_head list;
+ struct list_head tsgl_list; /* Link to TX SGL */
void *iv;
+ size_t aead_assoclen;
- struct af_alg_completion completion;
+ struct af_alg_completion completion; /* sync work queue */
- unsigned long used;
+ unsigned int inflight; /* Outstanding AIO ops */
+ size_t used; /* TX bytes sent to kernel */
+ size_t rcvused; /* total RX bytes to be processed by kernel */
- unsigned int len;
- bool more;
- bool merge;
- bool enc;
+ bool more; /* More data to be expected? */
+ bool merge; /* Merge new data into existing SG */
+ bool enc; /* Crypto operation: enc, dec */
- size_t aead_assoclen;
- struct aead_request aead_req;
+ unsigned int len; /* Length of allocated memory for this struct */
+ struct crypto_aead *aead_tfm;
};
+static DECLARE_WAIT_QUEUE_HEAD(aead_aio_finish_wait);
+
+#define MAX_SGL_ENTS ((4096 - sizeof(struct aead_tsgl)) / \
+ sizeof(struct scatterlist) - 1)
+
static inline int aead_sndbuf(struct sock *sk)
{
struct alg_sock *ask = alg_sk(sk);
@@ -78,9 +106,23 @@ static inline bool aead_writable(struct sock *sk)
return PAGE_SIZE <= aead_sndbuf(sk);
}
+static inline int aead_rcvbuf(struct sock *sk)
+{
+ struct alg_sock *ask = alg_sk(sk);
+ struct aead_ctx *ctx = ask->private;
+
+ return max_t(int, max_t(int, sk->sk_rcvbuf & PAGE_MASK, PAGE_SIZE) -
+ ctx->rcvused, 0);
+}
+
+static inline bool aead_readable(struct sock *sk)
+{
+ return PAGE_SIZE <= aead_rcvbuf(sk);
+}
+
static inline bool aead_sufficient_data(struct aead_ctx *ctx)
{
- unsigned as = crypto_aead_authsize(crypto_aead_reqtfm(&ctx->aead_req));
+ unsigned int as = crypto_aead_authsize(ctx->aead_tfm);
/*
* The minimum amount of memory needed for an AEAD cipher is
@@ -89,33 +131,166 @@ static inline bool aead_sufficient_data(struct aead_ctx *ctx)
return ctx->used >= ctx->aead_assoclen + (ctx->enc ? 0 : as);
}
-static void aead_reset_ctx(struct aead_ctx *ctx)
+static int aead_alloc_tsgl(struct sock *sk)
{
- struct aead_sg_list *sgl = &ctx->tsgl;
+ struct alg_sock *ask = alg_sk(sk);
+ struct aead_ctx *ctx = ask->private;
+ struct aead_tsgl *sgl;
+ struct scatterlist *sg = NULL;
- sg_init_table(sgl->sg, ALG_MAX_PAGES);
- sgl->cur = 0;
- ctx->used = 0;
- ctx->more = 0;
- ctx->merge = 0;
+ sgl = list_entry(ctx->tsgl_list.prev, struct aead_tsgl, list);
+ if (!list_empty(&ctx->tsgl_list))
+ sg = sgl->sg;
+
+ if (!sg || sgl->cur >= MAX_SGL_ENTS) {
+ sgl = sock_kmalloc(sk, sizeof(*sgl) +
+ sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1),
+ GFP_KERNEL);
+ if (!sgl)
+ return -ENOMEM;
+
+ sg_init_table(sgl->sg, MAX_SGL_ENTS + 1);
+ sgl->cur = 0;
+
+ if (sg)
+ sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg);
+
+ list_add_tail(&sgl->list, &ctx->tsgl_list);
+ }
+
+ return 0;
+}
+
+static unsigned int aead_count_tsgl(struct sock *sk, size_t bytes)
+{
+ struct alg_sock *ask = alg_sk(sk);
+ struct aead_ctx *ctx = ask->private;
+ struct aead_tsgl *sgl, *tmp;
+ unsigned int i;
+ unsigned int sgl_count = 0;
+
+ if (!bytes)
+ return 0;
+
+ list_for_each_entry_safe(sgl, tmp, &ctx->tsgl_list, list) {
+ struct scatterlist *sg = sgl->sg;
+
+ for (i = 0; i < sgl->cur; i++) {
+ sgl_count++;
+ if (sg[i].length >= bytes)
+ return sgl_count;
+
+ bytes -= sg[i].length;
+ }
+ }
+
+ return sgl_count;
+}
+
+static void aead_pull_tsgl(struct sock *sk, size_t used,
+ struct scatterlist *dst)
+{
+ struct alg_sock *ask = alg_sk(sk);
+ struct aead_ctx *ctx = ask->private;
+ struct aead_tsgl *sgl;
+ struct scatterlist *sg;
+ unsigned int i;
+
+ while (!list_empty(&ctx->tsgl_list)) {
+ sgl = list_first_entry(&ctx->tsgl_list, struct aead_tsgl,
+ list);
+ sg = sgl->sg;
+
+ for (i = 0; i < sgl->cur; i++) {
+ size_t plen = min_t(size_t, used, sg[i].length);
+ struct page *page = sg_page(sg + i);
+
+ if (!page)
+ continue;
+
+ /*
+ * Assumption: caller created aead_count_tsgl(len)
+ * SG entries in dst.
+ */
+ if (dst)
+ sg_set_page(dst + i, page, plen, sg[i].offset);
+
+ sg[i].length -= plen;
+ sg[i].offset += plen;
+
+ used -= plen;
+ ctx->used -= plen;
+
+ if (sg[i].length)
+ return;
+
+ if (!dst)
+ put_page(page);
+ sg_assign_page(sg + i, NULL);
+ }
+
+ list_del(&sgl->list);
+ sock_kfree_s(sk, sgl, sizeof(*sgl) + sizeof(sgl->sg[0]) *
+ (MAX_SGL_ENTS + 1));
+ }
+
+ if (!ctx->used)
+ ctx->merge = 0;
}
-static void aead_put_sgl(struct sock *sk)
+static void aead_free_areq_sgls(struct aead_async_req *areq)
{
+ struct sock *sk = areq->sk;
struct alg_sock *ask = alg_sk(sk);
struct aead_ctx *ctx = ask->private;
- struct aead_sg_list *sgl = &ctx->tsgl;
- struct scatterlist *sg = sgl->sg;
+ struct aead_rsgl *rsgl, *tmp;
+ struct scatterlist *tsgl;
+ struct scatterlist *sg;
unsigned int i;
- for (i = 0; i < sgl->cur; i++) {
- if (!sg_page(sg + i))
+ list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) {
+ ctx->rcvused -= rsgl->sg_num_bytes;
+ af_alg_free_sg(&rsgl->sgl);
+ list_del(&rsgl->list);
+ if (rsgl != &areq->first_rsgl)
+ sock_kfree_s(sk, rsgl, sizeof(*rsgl));
+ }
+
+ tsgl = areq->tsgl;
+ for_each_sg(tsgl, sg, areq->tsgl_entries, i) {
+ if (!sg_page(sg))
continue;
+ put_page(sg_page(sg));
+ }
+
+ if (areq->tsgl && areq->tsgl_entries)
+ sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl));
+}
+
+static int aead_wait_for_wmem(struct sock *sk, unsigned flags)
+{
+ DEFINE_WAIT_FUNC(wait, woken_wake_function);
+ int err = -ERESTARTSYS;
+ long timeout;
- put_page(sg_page(sg + i));
- sg_assign_page(sg + i, NULL);
+ if (flags & MSG_DONTWAIT)
+ return -EAGAIN;
+
+ sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
+
+ add_wait_queue(sk_sleep(sk), &wait);
+ for (;;) {
+ if (signal_pending(current))
+ break;
+ timeout = MAX_SCHEDULE_TIMEOUT;
+ if (sk_wait_event(sk, &timeout, aead_writable(sk), &wait)) {
+ err = 0;
+ break;
+ }
}
- aead_reset_ctx(ctx);
+ remove_wait_queue(sk_sleep(sk), &wait);
+
+ return err;
}
static void aead_wmem_wakeup(struct sock *sk)
@@ -147,6 +322,7 @@ static int aead_wait_for_data(struct sock *sk, unsigned flags)
return -EAGAIN;
sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
+
add_wait_queue(sk_sleep(sk), &wait);
for (;;) {
if (signal_pending(current))
@@ -170,8 +346,6 @@ static void aead_data_wakeup(struct sock *sk)
struct aead_ctx *ctx = ask->private;
struct socket_wq *wq;
- if (ctx->more)
- return;
if (!ctx->used)
return;
@@ -190,14 +364,13 @@ static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
struct sock *sk = sock->sk;
struct alg_sock *ask = alg_sk(sk);
struct aead_ctx *ctx = ask->private;
- unsigned ivsize =
- crypto_aead_ivsize(crypto_aead_reqtfm(&ctx->aead_req));
- struct aead_sg_list *sgl = &ctx->tsgl;
+ unsigned int ivsize = crypto_aead_ivsize(ctx->aead_tfm);
+ struct aead_tsgl *sgl;
struct af_alg_control con = {};
long copied = 0;
bool enc = 0;
bool init = 0;
- int err = -EINVAL;
+ int err = 0;
if (msg->msg_controllen) {
err = af_alg_cmsg_send(msg, &con);
@@ -221,8 +394,10 @@ static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
}
lock_sock(sk);
- if (!ctx->more && ctx->used)
+ if (!ctx->more && ctx->used) {
+ err = -EINVAL;
goto unlock;
+ }
if (init) {
ctx->enc = enc;
@@ -233,11 +408,14 @@ static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
}
while (size) {
+ struct scatterlist *sg;
size_t len = size;
- struct scatterlist *sg = NULL;
+ size_t plen;
/* use the existing memory in an allocated page */
if (ctx->merge) {
+ sgl = list_entry(ctx->tsgl_list.prev,
+ struct aead_tsgl, list);
sg = sgl->sg + sgl->cur - 1;
len = min_t(unsigned long, len,
PAGE_SIZE - sg->offset - sg->length);
@@ -258,57 +436,60 @@ static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
}
if (!aead_writable(sk)) {
- /* user space sent too much data */
- aead_put_sgl(sk);
- err = -EMSGSIZE;
- goto unlock;
+ err = aead_wait_for_wmem(sk, msg->msg_flags);
+ if (err)
+ goto unlock;
}
/* allocate a new page */
len = min_t(unsigned long, size, aead_sndbuf(sk));
- while (len) {
- size_t plen = 0;
- if (sgl->cur >= ALG_MAX_PAGES) {
- aead_put_sgl(sk);
- err = -E2BIG;
- goto unlock;
- }
+ err = aead_alloc_tsgl(sk);
+ if (err)
+ goto unlock;
+
+ sgl = list_entry(ctx->tsgl_list.prev, struct aead_tsgl,
+ list);
+ sg = sgl->sg;
+ if (sgl->cur)
+ sg_unmark_end(sg + sgl->cur - 1);
+
+ do {
+ unsigned int i = sgl->cur;
- sg = sgl->sg + sgl->cur;
plen = min_t(size_t, len, PAGE_SIZE);
- sg_assign_page(sg, alloc_page(GFP_KERNEL));
- err = -ENOMEM;
- if (!sg_page(sg))
+ sg_assign_page(sg + i, alloc_page(GFP_KERNEL));
+ if (!sg_page(sg + i)) {
+ err = -ENOMEM;
goto unlock;
+ }
- err = memcpy_from_msg(page_address(sg_page(sg)),
+ err = memcpy_from_msg(page_address(sg_page(sg + i)),
msg, plen);
if (err) {
- __free_page(sg_page(sg));
- sg_assign_page(sg, NULL);
+ __free_page(sg_page(sg + i));
+ sg_assign_page(sg + i, NULL);
goto unlock;
}
- sg->offset = 0;
- sg->length = plen;
+ sg[i].length = plen;
len -= plen;
ctx->used += plen;
copied += plen;
- sgl->cur++;
size -= plen;
- ctx->merge = plen & (PAGE_SIZE - 1);
- }
+ sgl->cur++;
+ } while (len && sgl->cur < MAX_SGL_ENTS);
+
+ if (!size)
+ sg_mark_end(sg + sgl->cur - 1);
+
+ ctx->merge = plen & (PAGE_SIZE - 1);
}
err = 0;
ctx->more = msg->msg_flags & MSG_MORE;
- if (!ctx->more && !aead_sufficient_data(ctx)) {
- aead_put_sgl(sk);
- err = -EMSGSIZE;
- }
unlock:
aead_data_wakeup(sk);
@@ -323,15 +504,12 @@ static ssize_t aead_sendpage(struct socket *sock, struct page *page,
struct sock *sk = sock->sk;
struct alg_sock *ask = alg_sk(sk);
struct aead_ctx *ctx = ask->private;
- struct aead_sg_list *sgl = &ctx->tsgl;
+ struct aead_tsgl *sgl;
int err = -EINVAL;
if (flags & MSG_SENDPAGE_NOTLAST)
flags |= MSG_MORE;
- if (sgl->cur >= ALG_MAX_PAGES)
- return -E2BIG;
-
lock_sock(sk);
if (!ctx->more && ctx->used)
goto unlock;
@@ -340,13 +518,22 @@ static ssize_t aead_sendpage(struct socket *sock, struct page *page,
goto done;
if (!aead_writable(sk)) {
- /* user space sent too much data */
- aead_put_sgl(sk);
- err = -EMSGSIZE;
- goto unlock;
+ err = aead_wait_for_wmem(sk, flags);
+ if (err)
+ goto unlock;
}
+ err = aead_alloc_tsgl(sk);
+ if (err)
+ goto unlock;
+
ctx->merge = 0;
+ sgl = list_entry(ctx->tsgl_list.prev, struct aead_tsgl, list);
+
+ if (sgl->cur)
+ sg_unmark_end(sgl->sg + sgl->cur - 1);
+
+ sg_mark_end(sgl->sg + sgl->cur);
get_page(page);
sg_set_page(sgl->sg + sgl->cur, page, size, offset);
@@ -357,11 +544,6 @@ static ssize_t aead_sendpage(struct socket *sock, struct page *page,
done:
ctx->more = flags & MSG_MORE;
- if (!ctx->more && !aead_sufficient_data(ctx)) {
- aead_put_sgl(sk);
- err = -EMSGSIZE;
- }
-
unlock:
aead_data_wakeup(sk);
release_sock(sk);
@@ -369,205 +551,56 @@ static ssize_t aead_sendpage(struct socket *sock, struct page *page,
return err ?: size;
}
-#define GET_ASYM_REQ(req, tfm) (struct aead_async_req *) \
- ((char *)req + sizeof(struct aead_request) + \
- crypto_aead_reqsize(tfm))
-
- #define GET_REQ_SIZE(tfm) sizeof(struct aead_async_req) + \
- crypto_aead_reqsize(tfm) + crypto_aead_ivsize(tfm) + \
- sizeof(struct aead_request)
-
static void aead_async_cb(struct crypto_async_request *_req, int err)
{
- struct sock *sk = _req->data;
+ struct aead_async_req *areq = _req->data;
+ struct sock *sk = areq->sk;
struct alg_sock *ask = alg_sk(sk);
struct aead_ctx *ctx = ask->private;
- struct crypto_aead *tfm = crypto_aead_reqtfm(&ctx->aead_req);
- struct aead_request *req = aead_request_cast(_req);
- struct aead_async_req *areq = GET_ASYM_REQ(req, tfm);
- struct scatterlist *sg = areq->tsgl;
- struct aead_async_rsgl *rsgl;
struct kiocb *iocb = areq->iocb;
- unsigned int i, reqlen = GET_REQ_SIZE(tfm);
-
- list_for_each_entry(rsgl, &areq->list, list) {
- af_alg_free_sg(&rsgl->sgl);
- if (rsgl != &areq->first_rsgl)
- sock_kfree_s(sk, rsgl, sizeof(*rsgl));
- }
-
- for (i = 0; i < areq->tsgls; i++)
- put_page(sg_page(sg + i));
-
- sock_kfree_s(sk, areq->tsgl, sizeof(*areq->tsgl) * areq->tsgls);
- sock_kfree_s(sk, req, reqlen);
- __sock_put(sk);
- iocb->ki_complete(iocb, err, err);
-}
-
-static int aead_recvmsg_async(struct socket *sock, struct msghdr *msg,
- int flags)
-{
- struct sock *sk = sock->sk;
- struct alg_sock *ask = alg_sk(sk);
- struct aead_ctx *ctx = ask->private;
- struct crypto_aead *tfm = crypto_aead_reqtfm(&ctx->aead_req);
- struct aead_async_req *areq;
- struct aead_request *req = NULL;
- struct aead_sg_list *sgl = &ctx->tsgl;
- struct aead_async_rsgl *last_rsgl = NULL, *rsgl;
- unsigned int as = crypto_aead_authsize(tfm);
- unsigned int i, reqlen = GET_REQ_SIZE(tfm);
- int err = -ENOMEM;
- unsigned long used;
- size_t outlen = 0;
- size_t usedpages = 0;
+ unsigned int resultlen;
lock_sock(sk);
- if (ctx->more) {
- err = aead_wait_for_data(sk, flags);
- if (err)
- goto unlock;
- }
-
- if (!aead_sufficient_data(ctx))
- goto unlock;
-
- used = ctx->used;
- if (ctx->enc)
- outlen = used + as;
- else
- outlen = used - as;
-
- req = sock_kmalloc(sk, reqlen, GFP_KERNEL);
- if (unlikely(!req))
- goto unlock;
-
- areq = GET_ASYM_REQ(req, tfm);
- memset(&areq->first_rsgl, '\0', sizeof(areq->first_rsgl));
- INIT_LIST_HEAD(&areq->list);
- areq->iocb = msg->msg_iocb;
- memcpy(areq->iv, ctx->iv, crypto_aead_ivsize(tfm));
- aead_request_set_tfm(req, tfm);
- aead_request_set_ad(req, ctx->aead_assoclen);
- aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
- aead_async_cb, sk);
- used -= ctx->aead_assoclen;
-
- /* take over all tx sgls from ctx */
- areq->tsgl = sock_kmalloc(sk,
- sizeof(*areq->tsgl) * max_t(u32, sgl->cur, 1),
- GFP_KERNEL);
- if (unlikely(!areq->tsgl))
- goto free;
- sg_init_table(areq->tsgl, max_t(u32, sgl->cur, 1));
- for (i = 0; i < sgl->cur; i++)
- sg_set_page(&areq->tsgl[i], sg_page(&sgl->sg[i]),
- sgl->sg[i].length, sgl->sg[i].offset);
+ BUG_ON(!ctx->inflight);
- areq->tsgls = sgl->cur;
+ /* Buffer size written by crypto operation. */
+ resultlen = areq->outlen;
- /* create rx sgls */
- while (outlen > usedpages && iov_iter_count(&msg->msg_iter)) {
- size_t seglen = min_t(size_t, iov_iter_count(&msg->msg_iter),
- (outlen - usedpages));
-
- if (list_empty(&areq->list)) {
- rsgl = &areq->first_rsgl;
-
- } else {
- rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
- if (unlikely(!rsgl)) {
- err = -ENOMEM;
- goto free;
- }
- }
- rsgl->sgl.npages = 0;
- list_add_tail(&rsgl->list, &areq->list);
-
- /* make one iovec available as scatterlist */
- err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
- if (err < 0)
- goto free;
-
- usedpages += err;
-
- /* chain the new scatterlist with previous one */
- if (last_rsgl)
- af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl);
-
- last_rsgl = rsgl;
-
- iov_iter_advance(&msg->msg_iter, err);
- }
+ aead_free_areq_sgls(areq);
+ sock_kfree_s(sk, areq, areq->areqlen);
+ __sock_put(sk);
+ ctx->inflight--;
- /* ensure output buffer is sufficiently large */
- if (usedpages < outlen) {
- err = -EINVAL;
- goto unlock;
- }
+ iocb->ki_complete(iocb, err ? err : resultlen, 0);
- aead_request_set_crypt(req, areq->tsgl, areq->first_rsgl.sgl.sg, used,
- areq->iv);
- err = ctx->enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
- if (err) {
- if (err == -EINPROGRESS) {
- sock_hold(sk);
- err = -EIOCBQUEUED;
- aead_reset_ctx(ctx);
- goto unlock;
- } else if (err == -EBADMSG) {
- aead_put_sgl(sk);
- }
- goto free;
- }
- aead_put_sgl(sk);
-
-free:
- list_for_each_entry(rsgl, &areq->list, list) {
- af_alg_free_sg(&rsgl->sgl);
- if (rsgl != &areq->first_rsgl)
- sock_kfree_s(sk, rsgl, sizeof(*rsgl));
- }
- if (areq->tsgl)
- sock_kfree_s(sk, areq->tsgl, sizeof(*areq->tsgl) * areq->tsgls);
- if (req)
- sock_kfree_s(sk, req, reqlen);
-unlock:
- aead_wmem_wakeup(sk);
release_sock(sk);
- return err ? err : outlen;
+
+ wake_up_interruptible(&aead_aio_finish_wait);
}
-static int aead_recvmsg_sync(struct socket *sock, struct msghdr *msg, int flags)
+static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, size_t ignored,
+ int flags)
{
struct sock *sk = sock->sk;
struct alg_sock *ask = alg_sk(sk);
struct aead_ctx *ctx = ask->private;
- unsigned as = crypto_aead_authsize(crypto_aead_reqtfm(&ctx->aead_req));
- struct aead_sg_list *sgl = &ctx->tsgl;
- struct aead_async_rsgl *last_rsgl = NULL;
- struct aead_async_rsgl *rsgl, *tmp;
- int err = -EINVAL;
- unsigned long used = 0;
- size_t outlen = 0;
- size_t usedpages = 0;
-
- lock_sock(sk);
+ struct crypto_aead *tfm = ctx->aead_tfm;
+ unsigned int as = crypto_aead_authsize(tfm);
+ unsigned int areqlen =
+ sizeof(struct aead_async_req) + crypto_aead_reqsize(tfm);
+ struct aead_async_req *areq;
+ struct aead_rsgl *last_rsgl = NULL;
+ int err = 0;
+ size_t used = 0; /* [in] TX bufs to be en/decrypted */
+ size_t outlen = 0; /* [out] RX bufs produced by kernel */
+ size_t usedpages = 0; /* [in] RX bufs to be used from user */
+ size_t processed = 0; /* [in] TX bufs to be consumed */
/*
- * Please see documentation of aead_request_set_crypt for the
- * description of the AEAD memory structure expected from the caller.
+ * Data length provided by caller via sendmsg/sendpage that has not
+ * yet been processed.
*/
-
- if (ctx->more) {
- err = aead_wait_for_data(sk, flags);
- if (err)
- goto unlock;
- }
-
- /* data length provided by caller via sendmsg/sendpage */
used = ctx->used;
/*
@@ -580,7 +613,7 @@ static int aead_recvmsg_sync(struct socket *sock, struct msghdr *msg, int flags)
* check here protects the kernel integrity.
*/
if (!aead_sufficient_data(ctx))
- goto unlock;
+ return -EINVAL;
/*
* Calculate the minimum output buffer size holding the result of the
@@ -601,84 +634,172 @@ static int aead_recvmsg_sync(struct socket *sock, struct msghdr *msg, int flags)
*/
used -= ctx->aead_assoclen;
- /* convert iovecs of output buffers into scatterlists */
+ /* Allocate cipher request for current operation. */
+ areq = sock_kmalloc(sk, areqlen, GFP_KERNEL);
+ if (unlikely(!areq))
+ return -ENOMEM;
+ areq->areqlen = areqlen;
+ areq->sk = sk;
+ INIT_LIST_HEAD(&areq->rsgl_list);
+ areq->tsgl = NULL;
+ areq->tsgl_entries = 0;
+
+ /* convert iovecs of output buffers into RX SGL */
while (outlen > usedpages && iov_iter_count(&msg->msg_iter)) {
- size_t seglen = min_t(size_t, iov_iter_count(&msg->msg_iter),
- (outlen - usedpages));
+ struct aead_rsgl *rsgl;
+ size_t seglen;
+
+ /* limit the amount of readable buffers */
+ if (!aead_readable(sk))
+ break;
- if (list_empty(&ctx->list)) {
- rsgl = &ctx->first_rsgl;
+ if (!ctx->used) {
+ err = aead_wait_for_data(sk, flags);
+ if (err)
+ goto free;
+ }
+
+ seglen = min_t(size_t, (outlen - usedpages),
+ iov_iter_count(&msg->msg_iter));
+
+ if (list_empty(&areq->rsgl_list)) {
+ rsgl = &areq->first_rsgl;
} else {
rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
if (unlikely(!rsgl)) {
err = -ENOMEM;
- goto unlock;
+ goto free;
}
}
+
rsgl->sgl.npages = 0;
- list_add_tail(&rsgl->list, &ctx->list);
+ list_add_tail(&rsgl->list, &areq->rsgl_list);
/* make one iovec available as scatterlist */
err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
if (err < 0)
- goto unlock;
- usedpages += err;
+ goto free;
+
/* chain the new scatterlist with previous one */
if (last_rsgl)
af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl);
last_rsgl = rsgl;
-
+ usedpages += err;
+ ctx->rcvused += err;
+ rsgl->sg_num_bytes = err;
iov_iter_advance(&msg->msg_iter, err);
}
- /* ensure output buffer is sufficiently large */
+ /*
+ * Ensure output buffer is sufficiently large. If the caller provides
+ * less buffer space, only use the relative required input size. This
+ * allows AIO operation where the caller sent all data to be processed
+ * and the AIO operation performs the operation on the different chunks
+ * of the input data.
+ */
if (usedpages < outlen) {
- err = -EINVAL;
- goto unlock;
- }
+ size_t less = outlen - usedpages;
- sg_mark_end(sgl->sg + sgl->cur - 1);
- aead_request_set_crypt(&ctx->aead_req, sgl->sg, ctx->first_rsgl.sgl.sg,
- used, ctx->iv);
- aead_request_set_ad(&ctx->aead_req, ctx->aead_assoclen);
+ if (used < less) {
+ err = -EINVAL;
+ goto free;
+ }
+ used -= less;
+ outlen -= less;
+ }
- err = af_alg_wait_for_completion(ctx->enc ?
- crypto_aead_encrypt(&ctx->aead_req) :
- crypto_aead_decrypt(&ctx->aead_req),
+ /*
+ * Create a per request TX SGL for this request which tracks the
+ * SG entries from the global TX SGL.
+ */
+ processed = used + ctx->aead_assoclen;
+ areq->tsgl_entries = aead_count_tsgl(sk, processed);
+ if (!areq->tsgl_entries)
+ areq->tsgl_entries = 1;
+ areq->tsgl = sock_kmalloc(sk, sizeof(*areq->tsgl) * areq->tsgl_entries,
+ GFP_KERNEL);
+ if (!areq->tsgl) {
+ err = -ENOMEM;
+ goto free;
+ }
+ sg_init_table(areq->tsgl, areq->tsgl_entries);
+ aead_pull_tsgl(sk, processed, areq->tsgl);
+
+ /* Initialize the crypto operation */
+ aead_request_set_crypt(&areq->aead_req, areq->tsgl,
+ areq->first_rsgl.sgl.sg, used, ctx->iv);
+ aead_request_set_ad(&areq->aead_req, ctx->aead_assoclen);
+ aead_request_set_tfm(&areq->aead_req, tfm);
+
+ if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) {
+ /* AIO operation */
+ areq->iocb = msg->msg_iocb;
+ aead_request_set_callback(&areq->aead_req,
+ CRYPTO_TFM_REQ_MAY_BACKLOG,
+ aead_async_cb, areq);
+ err = ctx->enc ? crypto_aead_encrypt(&areq->aead_req) :
+ crypto_aead_decrypt(&areq->aead_req);
+ } else {
+ /* Synchronous operation */
+ aead_request_set_callback(&areq->aead_req,
+ CRYPTO_TFM_REQ_MAY_BACKLOG,
+ af_alg_complete, &ctx->completion);
+ err = af_alg_wait_for_completion(ctx->enc ?
+ crypto_aead_encrypt(&areq->aead_req) :
+ crypto_aead_decrypt(&areq->aead_req),
&ctx->completion);
-
- if (err) {
- /* EBADMSG implies a valid cipher operation took place */
- if (err == -EBADMSG)
- aead_put_sgl(sk);
-
- goto unlock;
}
- aead_put_sgl(sk);
- err = 0;
+ /* AIO operation in progress */
+ if (err == -EINPROGRESS) {
+ sock_hold(sk);
+ ctx->inflight++;
-unlock:
- list_for_each_entry_safe(rsgl, tmp, &ctx->list, list) {
- af_alg_free_sg(&rsgl->sgl);
- list_del(&rsgl->list);
- if (rsgl != &ctx->first_rsgl)
- sock_kfree_s(sk, rsgl, sizeof(*rsgl));
+ /* Remember output size that will be generated. */
+ areq->outlen = outlen;
+
+ return -EIOCBQUEUED;
}
- INIT_LIST_HEAD(&ctx->list);
- aead_wmem_wakeup(sk);
- release_sock(sk);
+
+free:
+ aead_free_areq_sgls(areq);
+ if (areq)
+ sock_kfree_s(sk, areq, areqlen);
return err ? err : outlen;
}
-static int aead_recvmsg(struct socket *sock, struct msghdr *msg, size_t ignored,
- int flags)
+static int aead_recvmsg(struct socket *sock, struct msghdr *msg,
+ size_t ignored, int flags)
{
- return (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) ?
- aead_recvmsg_async(sock, msg, flags) :
- aead_recvmsg_sync(sock, msg, flags);
+ struct sock *sk = sock->sk;
+ int ret = 0;
+
+ lock_sock(sk);
+ while (iov_iter_count(&msg->msg_iter)) {
+ int err = _aead_recvmsg(sock, msg, ignored, flags);
+
+ /*
+ * This error covers -EIOCBQUEUED which implies that we can
+ * only handle one AIO request. If the caller wants to have
+ * multiple AIO requests in parallel, he must make multiple
+ * separate AIO calls.
+ */
+ if (err < 0) {
+ ret = err;
+ goto out;
+ }
+ if (!err)
+ goto out;
+
+ ret += err;
+ }
+
+out:
+ aead_wmem_wakeup(sk);
+ release_sock(sk);
+ return ret;
}
static unsigned int aead_poll(struct file *file, struct socket *sock,
@@ -687,10 +808,9 @@ static unsigned int aead_poll(struct file *file, struct socket *sock,
struct sock *sk = sock->sk;
struct alg_sock *ask = alg_sk(sk);
struct aead_ctx *ctx = ask->private;
- unsigned int mask;
+ unsigned int mask = 0;
sock_poll_wait(file, sk_sleep(sk), wait);
- mask = 0;
if (!ctx->more)
mask |= POLLIN | POLLRDNORM;
@@ -747,11 +867,14 @@ static void aead_sock_destruct(struct sock *sk)
{
struct alg_sock *ask = alg_sk(sk);
struct aead_ctx *ctx = ask->private;
- unsigned int ivlen = crypto_aead_ivsize(
- crypto_aead_reqtfm(&ctx->aead_req));
+ unsigned int ivlen = crypto_aead_ivsize(ctx->aead_tfm);
WARN_ON(atomic_read(&sk->sk_refcnt) != 0);
- aead_put_sgl(sk);
+
+ /* Suspend caller if AIO operations are in flight. */
+ wait_event_interruptible(aead_aio_finish_wait, (ctx->inflight == 0));
+
+ aead_pull_tsgl(sk, ctx->used, NULL);
sock_kzfree_s(sk, ctx->iv, ivlen);
sock_kfree_s(sk, ctx, ctx->len);
af_alg_release_parent(sk);
@@ -761,7 +884,7 @@ static int aead_accept_parent(void *private, struct sock *sk)
{
struct aead_ctx *ctx;
struct alg_sock *ask = alg_sk(sk);
- unsigned int len = sizeof(*ctx) + crypto_aead_reqsize(private);
+ unsigned int len = sizeof(*ctx);
unsigned int ivlen = crypto_aead_ivsize(private);
ctx = sock_kmalloc(sk, len, GFP_KERNEL);
@@ -776,23 +899,20 @@ static int aead_accept_parent(void *private, struct sock *sk)
}
memset(ctx->iv, 0, ivlen);
+ INIT_LIST_HEAD(&ctx->tsgl_list);
ctx->len = len;
ctx->used = 0;
+ ctx->rcvused = 0;
ctx->more = 0;
ctx->merge = 0;
ctx->enc = 0;
- ctx->tsgl.cur = 0;
+ ctx->inflight = 0;
ctx->aead_assoclen = 0;
af_alg_init_completion(&ctx->completion);
- sg_init_table(ctx->tsgl.sg, ALG_MAX_PAGES);
- INIT_LIST_HEAD(&ctx->list);
+ ctx->aead_tfm = private;
ask->private = ctx;
- aead_request_set_tfm(&ctx->aead_req, private);
- aead_request_set_callback(&ctx->aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
- af_alg_complete, &ctx->completion);
-
sk->sk_destruct = aead_sock_destruct;
return 0;
--
2.9.3
^ permalink raw reply related
* [PATCH v6 0/2] crypto: AF_ALG memory management fix
From: Stephan Müller @ 2017-04-01 20:50 UTC (permalink / raw)
To: herbert; +Cc: linux-crypto
Hi Herbert,
Changes v6:
* port to 4.11-rc1 (a header file was included from upstream)
* Limit the RX SGL in an identical fashion as the TX SGL size is limited by
adding [skcipher|aead]_readable which is an equivalent to the existing
[skcipher|aead]_writable. To ensure that the limit is enforced, ctx->rcvused
is added which tracks the RX SGL memory currently in use for the context
(i.e. covering all requests that are currently processed).
* Add an outer loop to the recvmsg operation to allow continuous recvmsg
operation in parallel to a continuous sendmsg operation.
Changes v5:
* use list_for_each_entry_safe in aead_count_tsgl
Changes v4:
* replace ctx->processed with a maintenance of a private recvmsg
TX SGL
* algif_skcipher: rename skcipher_sg_list into skcipher_tsgl to make
it consistent with algif_aead to prepare for a code consolidation
Changes v3:
* in *_pull_tsgl: make sure ctx->processed cannot be less than zero
* perform fuzzing of all input parameters with bogus values
Changes v2:
* import fix from Harsh Jain <harsh@chelsio.com> to remove SG
from list before freeing
* fix return code used for ki_complete to match AIO behavior
with sync behavior
* rename variable list -> tsgl_list
* update the algif_aead patch to include a dynamic TX SGL
allocation similar to what algif_skcipher does. This allows
concurrent continuous read/write operations to the extent
you requested. Although I have not implemented "pairs of
TX/RX SGLs" as I think that is even more overhead, the
implementation conceptually defines such pairs. The recvmsg
call defines how much from the input data is processed.
The caller can have arbitrary number of sendmsg calls
where the data is added to the TX SGL before an recvmsg
asks the kernel to process a given amount (or all) of the
TX SGL.
With the changes, you will see a lot of code duplication now
as I deliberately tried to use the same struct and variable names,
the same function names and even the same oder of functions.
If you agree to this patch, I volunteer to provide a followup
patch that will extract the code duplication into common
functions.
Please find attached memory management updates to
- simplify the code: the old AIO memory management is very
complex and seemingly very fragile -- the update now
eliminates all reported bugs in the skcipher and AEAD
interfaces which allowed the kernel to be crashed by
an unprivileged user
- streamline the code: there is one code path for AIO and sync
operation; the code between algif_skcipher and algif_aead
is very similar (if that patch set is accepted, I volunteer
to reduce code duplication by moving service operations
into af_alg.c and to further unify the TX SGL handling)
- unify the AIO and sync operation which only differ in the
kernel crypto API callback and whether to wait for the
crypto operation or not
- fix all reported bugs regarding the handling of multiple
IOCBs.
The following testing was performed:
- stress testing to verify that no memleaks exist
- testing using Tadeusz Struck AIO test tool (see
https://github.com/tstruk/afalg_async_test) -- the AEAD test
is not applicable any more due to the changed user space
interface; the skcipher test works once the user space
interface change is honored in the test code
- using the libkcapi test suite, all tests including the
originally failing ones (AIO with multiple IOCBs) work now --
the current libkcapi code artificially limits the AEAD
operation to one IOCB. After altering the libkcapi code
to allow multiple IOCBs, the testing works flawless.
Stephan Mueller (2):
crypto: skcipher AF_ALG - overhaul memory management
crypto: aead AF_ALG - overhaul memory management
crypto/algif_aead.c | 762 +++++++++++++++++++++++++++
+--------------------
crypto/algif_skcipher.c | 582 ++++++++++++++++++------------------
2 files changed, 739 insertions(+), 605 deletions(-)
--
2.9.3
^ permalink raw reply
* [PATCH v6 1/2] crypto: skcipher AF_ALG - overhaul memory management
From: Stephan Müller @ 2017-04-01 20:52 UTC (permalink / raw)
To: herbert; +Cc: linux-crypto
In-Reply-To: <2473716.zZb2vDUueJ@positron.chronox.de>
The updated memory management is described in the top part of the code.
As one benefit of the changed memory management, the AIO and synchronous
operation is now implemented in one common function. The AF_ALG
operation uses the async kernel crypto API interface for each cipher
operation. Thus, the only difference between the AIO and sync operation
types visible from user space is:
1. the callback function to be invoked when the asynchronous operation
is completed
2. whether to wait for the completion of the kernel crypto API operation
or not
In addition, the code structure is adjusted to match the structure of
algif_aead for easier code assessment.
The user space interface changed slightly as follows: the old AIO
operation returned zero upon success and < 0 in case of an error to user
space. As all other AF_ALG interfaces (including the sync skcipher
interface) returned the number of processed bytes upon success and < 0
in case of an error, the new skcipher interface (regardless of AIO or
sync) returns the number of processed bytes in case of success.
Signed-off-by: Stephan Mueller <smueller@chronox.de>
---
crypto/algif_skcipher.c | 582 +++++++++++++++++++++++++-----------------------
1 file changed, 298 insertions(+), 284 deletions(-)
diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c
index 43839b0..0f086e1 100644
--- a/crypto/algif_skcipher.c
+++ b/crypto/algif_skcipher.c
@@ -10,6 +10,21 @@
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
+ * The following concept of the memory management is used:
+ *
+ * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is
+ * filled by user space with the data submitted via sendpage/sendmsg. Filling
+ * up the TX SGL does not cause a crypto operation -- the data will only be
+ * tracked by the kernel. Upon receipt of one recvmsg call, the caller must
+ * provide a buffer which is tracked with the RX SGL.
+ *
+ * During the processing of the recvmsg operation, the cipher request is
+ * allocated and prepared. As part of the recvmsg operation, the processed
+ * TX buffers are extracted from the TX SGL into a separate SGL.
+ *
+ * After the completion of the crypto operation, the RX SGL and the cipher
+ * request is released. The extracted TX SGL parts are released together with
+ * the RX SGL release.
*/
#include <crypto/scatterwalk.h>
@@ -24,109 +39,97 @@
#include <linux/net.h>
#include <net/sock.h>
-struct skcipher_sg_list {
+struct skcipher_tsgl {
struct list_head list;
-
int cur;
-
struct scatterlist sg[0];
};
+struct skcipher_rsgl {
+ struct af_alg_sgl sgl;
+ struct list_head list;
+ size_t sg_num_bytes;
+};
+
+struct skcipher_async_req {
+ struct kiocb *iocb;
+ struct sock *sk;
+
+ struct skcipher_rsgl first_sgl;
+ struct list_head rsgl_list;
+
+ struct scatterlist *tsgl;
+ unsigned int tsgl_entries;
+
+ unsigned int areqlen;
+ struct skcipher_request req;
+};
+
struct skcipher_tfm {
struct crypto_skcipher *skcipher;
bool has_key;
};
struct skcipher_ctx {
- struct list_head tsgl;
- struct af_alg_sgl rsgl;
+ struct list_head tsgl_list;
void *iv;
struct af_alg_completion completion;
- atomic_t inflight;
+ unsigned int inflight;
size_t used;
+ size_t rcvused;
- unsigned int len;
bool more;
bool merge;
bool enc;
- struct skcipher_request req;
-};
-
-struct skcipher_async_rsgl {
- struct af_alg_sgl sgl;
- struct list_head list;
+ unsigned int len;
};
-struct skcipher_async_req {
- struct kiocb *iocb;
- struct skcipher_async_rsgl first_sgl;
- struct list_head list;
- struct scatterlist *tsg;
- atomic_t *inflight;
- struct skcipher_request req;
-};
+static DECLARE_WAIT_QUEUE_HEAD(skcipher_aio_finish_wait);
-#define MAX_SGL_ENTS ((4096 - sizeof(struct skcipher_sg_list)) / \
+#define MAX_SGL_ENTS ((4096 - sizeof(struct skcipher_tsgl)) / \
sizeof(struct scatterlist) - 1)
-static void skcipher_free_async_sgls(struct skcipher_async_req *sreq)
+static inline int skcipher_sndbuf(struct sock *sk)
{
- struct skcipher_async_rsgl *rsgl, *tmp;
- struct scatterlist *sgl;
- struct scatterlist *sg;
- int i, n;
-
- list_for_each_entry_safe(rsgl, tmp, &sreq->list, list) {
- af_alg_free_sg(&rsgl->sgl);
- if (rsgl != &sreq->first_sgl)
- kfree(rsgl);
- }
- sgl = sreq->tsg;
- n = sg_nents(sgl);
- for_each_sg(sgl, sg, n, i)
- put_page(sg_page(sg));
+ struct alg_sock *ask = alg_sk(sk);
+ struct skcipher_ctx *ctx = ask->private;
- kfree(sreq->tsg);
+ return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) -
+ ctx->used, 0);
}
-static void skcipher_async_cb(struct crypto_async_request *req, int err)
+static inline bool skcipher_writable(struct sock *sk)
{
- struct skcipher_async_req *sreq = req->data;
- struct kiocb *iocb = sreq->iocb;
-
- atomic_dec(sreq->inflight);
- skcipher_free_async_sgls(sreq);
- kzfree(sreq);
- iocb->ki_complete(iocb, err, err);
+ return PAGE_SIZE <= skcipher_sndbuf(sk);
}
-static inline int skcipher_sndbuf(struct sock *sk)
+static inline int skcipher_rcvbuf(struct sock *sk)
{
struct alg_sock *ask = alg_sk(sk);
struct skcipher_ctx *ctx = ask->private;
- return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) -
- ctx->used, 0);
+ return max_t(int, max_t(int, sk->sk_rcvbuf & PAGE_MASK, PAGE_SIZE) -
+ ctx->rcvused, 0);
}
-static inline bool skcipher_writable(struct sock *sk)
+static inline bool skcipher_readable(struct sock *sk)
{
- return PAGE_SIZE <= skcipher_sndbuf(sk);
+ return PAGE_SIZE <= skcipher_rcvbuf(sk);
}
-static int skcipher_alloc_sgl(struct sock *sk)
+static int skcipher_alloc_tsgl(struct sock *sk)
{
struct alg_sock *ask = alg_sk(sk);
struct skcipher_ctx *ctx = ask->private;
- struct skcipher_sg_list *sgl;
+ struct skcipher_tsgl *sgl;
struct scatterlist *sg = NULL;
- sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
- if (!list_empty(&ctx->tsgl))
+ sgl = list_entry(ctx->tsgl_list.prev, struct skcipher_tsgl, list);
+ if (!list_empty(&ctx->tsgl_list))
sg = sgl->sg;
if (!sg || sgl->cur >= MAX_SGL_ENTS) {
@@ -142,31 +145,66 @@ static int skcipher_alloc_sgl(struct sock *sk)
if (sg)
sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg);
- list_add_tail(&sgl->list, &ctx->tsgl);
+ list_add_tail(&sgl->list, &ctx->tsgl_list);
}
return 0;
}
-static void skcipher_pull_sgl(struct sock *sk, size_t used, int put)
+static unsigned int skcipher_count_tsgl(struct sock *sk, size_t bytes)
{
struct alg_sock *ask = alg_sk(sk);
struct skcipher_ctx *ctx = ask->private;
- struct skcipher_sg_list *sgl;
+ struct skcipher_tsgl *sgl, *tmp;
+ unsigned int i;
+ unsigned int sgl_count = 0;
+
+ if (!bytes)
+ return 0;
+
+ list_for_each_entry_safe(sgl, tmp, &ctx->tsgl_list, list) {
+ struct scatterlist *sg = sgl->sg;
+
+ for (i = 0; i < sgl->cur; i++) {
+ sgl_count++;
+ if (sg[i].length >= bytes)
+ return sgl_count;
+
+ bytes -= sg[i].length;
+ }
+ }
+
+ return sgl_count;
+}
+
+static void skcipher_pull_tsgl(struct sock *sk, size_t used,
+ struct scatterlist *dst)
+{
+ struct alg_sock *ask = alg_sk(sk);
+ struct skcipher_ctx *ctx = ask->private;
+ struct skcipher_tsgl *sgl;
struct scatterlist *sg;
- int i;
+ unsigned int i;
- while (!list_empty(&ctx->tsgl)) {
- sgl = list_first_entry(&ctx->tsgl, struct skcipher_sg_list,
+ while (!list_empty(&ctx->tsgl_list)) {
+ sgl = list_first_entry(&ctx->tsgl_list, struct skcipher_tsgl,
list);
sg = sgl->sg;
for (i = 0; i < sgl->cur; i++) {
size_t plen = min_t(size_t, used, sg[i].length);
+ struct page *page = sg_page(sg + i);
- if (!sg_page(sg + i))
+ if (!page)
continue;
+ /*
+ * Assumption: caller created skcipher_count_tsgl(len)
+ * SG entries in dst.
+ */
+ if (dst)
+ sg_set_page(dst + i, page, plen, sg[i].offset);
+
sg[i].length -= plen;
sg[i].offset += plen;
@@ -175,27 +213,48 @@ static void skcipher_pull_sgl(struct sock *sk, size_t used, int put)
if (sg[i].length)
return;
- if (put)
- put_page(sg_page(sg + i));
+
+ if (!dst)
+ put_page(page);
sg_assign_page(sg + i, NULL);
}
list_del(&sgl->list);
- sock_kfree_s(sk, sgl,
- sizeof(*sgl) + sizeof(sgl->sg[0]) *
- (MAX_SGL_ENTS + 1));
+ sock_kfree_s(sk, sgl, sizeof(*sgl) + sizeof(sgl->sg[0]) *
+ (MAX_SGL_ENTS + 1));
}
if (!ctx->used)
ctx->merge = 0;
}
-static void skcipher_free_sgl(struct sock *sk)
+static void skcipher_free_areq_sgls(struct skcipher_async_req *areq)
{
+ struct sock *sk = areq->sk;
struct alg_sock *ask = alg_sk(sk);
struct skcipher_ctx *ctx = ask->private;
+ struct skcipher_rsgl *rsgl, *tmp;
+ struct scatterlist *tsgl;
+ struct scatterlist *sg;
+ unsigned int i;
+
+ list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) {
+ ctx->rcvused -= rsgl->sg_num_bytes;
+ af_alg_free_sg(&rsgl->sgl);
+ list_del(&rsgl->list);
+ if (rsgl != &areq->first_sgl)
+ sock_kfree_s(sk, rsgl, sizeof(*rsgl));
+ }
+
+ tsgl = areq->tsgl;
+ for_each_sg(tsgl, sg, areq->tsgl_entries, i) {
+ if (!sg_page(sg))
+ continue;
+ put_page(sg_page(sg));
+ }
- skcipher_pull_sgl(sk, ctx->used, 1);
+ if (areq->tsgl && areq->tsgl_entries)
+ sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl));
}
static int skcipher_wait_for_wmem(struct sock *sk, unsigned flags)
@@ -302,7 +361,7 @@ static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg,
struct skcipher_tfm *skc = pask->private;
struct crypto_skcipher *tfm = skc->skcipher;
unsigned ivsize = crypto_skcipher_ivsize(tfm);
- struct skcipher_sg_list *sgl;
+ struct skcipher_tsgl *sgl;
struct af_alg_control con = {};
long copied = 0;
bool enc = 0;
@@ -349,8 +408,8 @@ static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg,
size_t plen;
if (ctx->merge) {
- sgl = list_entry(ctx->tsgl.prev,
- struct skcipher_sg_list, list);
+ sgl = list_entry(ctx->tsgl_list.prev,
+ struct skcipher_tsgl, list);
sg = sgl->sg + sgl->cur - 1;
len = min_t(unsigned long, len,
PAGE_SIZE - sg->offset - sg->length);
@@ -379,11 +438,12 @@ static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg,
len = min_t(unsigned long, len, skcipher_sndbuf(sk));
- err = skcipher_alloc_sgl(sk);
+ err = skcipher_alloc_tsgl(sk);
if (err)
goto unlock;
- sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
+ sgl = list_entry(ctx->tsgl_list.prev, struct skcipher_tsgl,
+ list);
sg = sgl->sg;
if (sgl->cur)
sg_unmark_end(sg + sgl->cur - 1);
@@ -435,7 +495,7 @@ static ssize_t skcipher_sendpage(struct socket *sock, struct page *page,
struct sock *sk = sock->sk;
struct alg_sock *ask = alg_sk(sk);
struct skcipher_ctx *ctx = ask->private;
- struct skcipher_sg_list *sgl;
+ struct skcipher_tsgl *sgl;
int err = -EINVAL;
if (flags & MSG_SENDPAGE_NOTLAST)
@@ -454,12 +514,12 @@ static ssize_t skcipher_sendpage(struct socket *sock, struct page *page,
goto unlock;
}
- err = skcipher_alloc_sgl(sk);
+ err = skcipher_alloc_tsgl(sk);
if (err)
goto unlock;
ctx->merge = 0;
- sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
+ sgl = list_entry(ctx->tsgl_list.prev, struct skcipher_tsgl, list);
if (sgl->cur)
sg_unmark_end(sgl->sg + sgl->cur - 1);
@@ -480,25 +540,36 @@ static ssize_t skcipher_sendpage(struct socket *sock, struct page *page,
return err ?: size;
}
-static int skcipher_all_sg_nents(struct skcipher_ctx *ctx)
+static void skcipher_async_cb(struct crypto_async_request *req, int err)
{
- struct skcipher_sg_list *sgl;
- struct scatterlist *sg;
- int nents = 0;
+ struct skcipher_async_req *areq = req->data;
+ struct sock *sk = areq->sk;
+ struct alg_sock *ask = alg_sk(sk);
+ struct skcipher_ctx *ctx = ask->private;
+ struct kiocb *iocb = areq->iocb;
+ unsigned int resultlen;
- list_for_each_entry(sgl, &ctx->tsgl, list) {
- sg = sgl->sg;
+ lock_sock(sk);
- while (!sg->length)
- sg++;
+ BUG_ON(!ctx->inflight);
- nents += sg_nents(sg);
- }
- return nents;
+ /* Buffer size written by crypto operation. */
+ resultlen = areq->req.cryptlen;
+
+ skcipher_free_areq_sgls(areq);
+ sock_kfree_s(sk, areq, areq->areqlen);
+ __sock_put(sk);
+ ctx->inflight--;
+
+ iocb->ki_complete(iocb, err ? err : resultlen, 0);
+
+ release_sock(sk);
+
+ wake_up_interruptible(&skcipher_aio_finish_wait);
}
-static int skcipher_recvmsg_async(struct socket *sock, struct msghdr *msg,
- int flags)
+static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg,
+ size_t ignored, int flags)
{
struct sock *sk = sock->sk;
struct alg_sock *ask = alg_sk(sk);
@@ -507,215 +578,169 @@ static int skcipher_recvmsg_async(struct socket *sock, struct msghdr *msg,
struct skcipher_ctx *ctx = ask->private;
struct skcipher_tfm *skc = pask->private;
struct crypto_skcipher *tfm = skc->skcipher;
- struct skcipher_sg_list *sgl;
- struct scatterlist *sg;
- struct skcipher_async_req *sreq;
- struct skcipher_request *req;
- struct skcipher_async_rsgl *last_rsgl = NULL;
- unsigned int txbufs = 0, len = 0, tx_nents;
- unsigned int reqsize = crypto_skcipher_reqsize(tfm);
- unsigned int ivsize = crypto_skcipher_ivsize(tfm);
- int err = -ENOMEM;
- bool mark = false;
- char *iv;
-
- sreq = kzalloc(sizeof(*sreq) + reqsize + ivsize, GFP_KERNEL);
- if (unlikely(!sreq))
- goto out;
-
- req = &sreq->req;
- iv = (char *)(req + 1) + reqsize;
- sreq->iocb = msg->msg_iocb;
- INIT_LIST_HEAD(&sreq->list);
- sreq->inflight = &ctx->inflight;
-
- lock_sock(sk);
- tx_nents = skcipher_all_sg_nents(ctx);
- sreq->tsg = kcalloc(tx_nents, sizeof(*sg), GFP_KERNEL);
- if (unlikely(!sreq->tsg))
- goto unlock;
- sg_init_table(sreq->tsg, tx_nents);
- memcpy(iv, ctx->iv, ivsize);
- skcipher_request_set_tfm(req, tfm);
- skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP,
- skcipher_async_cb, sreq);
+ unsigned int bs = crypto_skcipher_blocksize(tfm);
+ unsigned int areqlen = sizeof(struct skcipher_async_req) +
+ crypto_skcipher_reqsize(tfm);
+ struct skcipher_async_req *areq;
+ struct skcipher_rsgl *last_rsgl = NULL;
+ int err = 0;
+ size_t len = 0;
- while (iov_iter_count(&msg->msg_iter)) {
- struct skcipher_async_rsgl *rsgl;
- int used;
+ /* Allocate cipher request for current operation. */
+ areq = sock_kmalloc(sk, areqlen, GFP_KERNEL);
+ if (unlikely(!areq))
+ return -ENOMEM;
+ areq->areqlen = areqlen;
+ areq->sk = sk;
+ INIT_LIST_HEAD(&areq->rsgl_list);
+ areq->tsgl = NULL;
+ areq->tsgl_entries = 0;
+
+ /* convert iovecs of output buffers into RX SGL */
+ while (len < ctx->used && iov_iter_count(&msg->msg_iter)) {
+ struct skcipher_rsgl *rsgl;
+ size_t seglen;
+
+ /* limit the amount of readable buffers */
+ if (!skcipher_readable(sk))
+ break;
if (!ctx->used) {
err = skcipher_wait_for_data(sk, flags);
if (err)
goto free;
}
- sgl = list_first_entry(&ctx->tsgl,
- struct skcipher_sg_list, list);
- sg = sgl->sg;
- while (!sg->length)
- sg++;
-
- used = min_t(unsigned long, ctx->used,
- iov_iter_count(&msg->msg_iter));
- used = min_t(unsigned long, used, sg->length);
-
- if (txbufs == tx_nents) {
- struct scatterlist *tmp;
- int x;
- /* Ran out of tx slots in async request
- * need to expand */
- tmp = kcalloc(tx_nents * 2, sizeof(*tmp),
- GFP_KERNEL);
- if (!tmp) {
- err = -ENOMEM;
- goto free;
- }
+ seglen = min_t(size_t, ctx->used,
+ iov_iter_count(&msg->msg_iter));
- sg_init_table(tmp, tx_nents * 2);
- for (x = 0; x < tx_nents; x++)
- sg_set_page(&tmp[x], sg_page(&sreq->tsg[x]),
- sreq->tsg[x].length,
- sreq->tsg[x].offset);
- kfree(sreq->tsg);
- sreq->tsg = tmp;
- tx_nents *= 2;
- mark = true;
- }
- /* Need to take over the tx sgl from ctx
- * to the asynch req - these sgls will be freed later */
- sg_set_page(sreq->tsg + txbufs++, sg_page(sg), sg->length,
- sg->offset);
-
- if (list_empty(&sreq->list)) {
- rsgl = &sreq->first_sgl;
- list_add_tail(&rsgl->list, &sreq->list);
+ if (list_empty(&areq->rsgl_list)) {
+ rsgl = &areq->first_sgl;
} else {
- rsgl = kmalloc(sizeof(*rsgl), GFP_KERNEL);
+ rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
if (!rsgl) {
err = -ENOMEM;
goto free;
}
- list_add_tail(&rsgl->list, &sreq->list);
}
- used = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, used);
- err = used;
- if (used < 0)
+ rsgl->sgl.npages = 0;
+ list_add_tail(&rsgl->list, &areq->rsgl_list);
+
+ /* make one iovec available as scatterlist */
+ err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
+ if (err < 0)
goto free;
+
+ /* chain the new scatterlist with previous one */
if (last_rsgl)
af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl);
last_rsgl = rsgl;
- len += used;
- skcipher_pull_sgl(sk, used, 0);
- iov_iter_advance(&msg->msg_iter, used);
+ len += err;
+ ctx->rcvused =+ err;
+ rsgl->sg_num_bytes = err;
+ iov_iter_advance(&msg->msg_iter, err);
}
- if (mark)
- sg_mark_end(sreq->tsg + txbufs - 1);
+ /* Process only as much RX buffers for which we have TX data */
+ if (len > ctx->used)
+ len = ctx->used;
+
+ /*
+ * If more buffers are to be expected to be processed, process only
+ * full block size buffers.
+ */
+ if (ctx->more || len < ctx->used)
+ len -= len % bs;
+
+ /*
+ * Create a per request TX SGL for this request which tracks the
+ * SG entries from the global TX SGL.
+ */
+ areq->tsgl_entries = skcipher_count_tsgl(sk, len);
+ if (!areq->tsgl_entries)
+ areq->tsgl_entries = 1;
+ areq->tsgl = sock_kmalloc(sk, sizeof(*areq->tsgl) * areq->tsgl_entries,
+ GFP_KERNEL);
+ if (!areq->tsgl) {
+ err = -ENOMEM;
+ goto free;
+ }
+ sg_init_table(areq->tsgl, areq->tsgl_entries);
+ skcipher_pull_tsgl(sk, len, areq->tsgl);
+
+ /* Initialize the crypto operation */
+ skcipher_request_set_tfm(&areq->req, tfm);
+ skcipher_request_set_crypt(&areq->req, areq->tsgl,
+ areq->first_sgl.sgl.sg, len, ctx->iv);
+
+ if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) {
+ /* AIO operation */
+ areq->iocb = msg->msg_iocb;
+ skcipher_request_set_callback(&areq->req,
+ CRYPTO_TFM_REQ_MAY_SLEEP,
+ skcipher_async_cb, areq);
+ err = ctx->enc ? crypto_skcipher_encrypt(&areq->req) :
+ crypto_skcipher_decrypt(&areq->req);
+ } else {
+ /* Synchronous operation */
+ skcipher_request_set_callback(&areq->req,
+ CRYPTO_TFM_REQ_MAY_SLEEP |
+ CRYPTO_TFM_REQ_MAY_BACKLOG,
+ af_alg_complete,
+ &ctx->completion);
+ err = af_alg_wait_for_completion(ctx->enc ?
+ crypto_skcipher_encrypt(&areq->req) :
+ crypto_skcipher_decrypt(&areq->req),
+ &ctx->completion);
+ }
- skcipher_request_set_crypt(req, sreq->tsg, sreq->first_sgl.sgl.sg,
- len, iv);
- err = ctx->enc ? crypto_skcipher_encrypt(req) :
- crypto_skcipher_decrypt(req);
+ /* AIO operation in progress */
if (err == -EINPROGRESS) {
- atomic_inc(&ctx->inflight);
- err = -EIOCBQUEUED;
- sreq = NULL;
- goto unlock;
+ sock_hold(sk);
+ ctx->inflight++;
+ return -EIOCBQUEUED;
}
+
free:
- skcipher_free_async_sgls(sreq);
-unlock:
- skcipher_wmem_wakeup(sk);
- release_sock(sk);
- kzfree(sreq);
-out:
- return err;
+ skcipher_free_areq_sgls(areq);
+ if (areq)
+ sock_kfree_s(sk, areq, areqlen);
+
+ return err ? err : len;
}
-static int skcipher_recvmsg_sync(struct socket *sock, struct msghdr *msg,
- int flags)
+static int skcipher_recvmsg(struct socket *sock, struct msghdr *msg,
+ size_t ignored, int flags)
{
struct sock *sk = sock->sk;
- struct alg_sock *ask = alg_sk(sk);
- struct sock *psk = ask->parent;
- struct alg_sock *pask = alg_sk(psk);
- struct skcipher_ctx *ctx = ask->private;
- struct skcipher_tfm *skc = pask->private;
- struct crypto_skcipher *tfm = skc->skcipher;
- unsigned bs = crypto_skcipher_blocksize(tfm);
- struct skcipher_sg_list *sgl;
- struct scatterlist *sg;
- int err = -EAGAIN;
- int used;
- long copied = 0;
+ int ret = 0;
lock_sock(sk);
- while (msg_data_left(msg)) {
- if (!ctx->used) {
- err = skcipher_wait_for_data(sk, flags);
- if (err)
- goto unlock;
+ while (iov_iter_count(&msg->msg_iter)) {
+ int err = _skcipher_recvmsg(sock, msg, ignored, flags);
+
+ /*
+ * This error covers -EIOCBQUEUED which implies that we can
+ * only handle one AIO request. If the caller wants to have
+ * multiple AIO requests in parallel, he must make multiple
+ * separate AIO calls.
+ */
+ if (err < 0) {
+ ret = err;
+ goto out;
}
+ if (!err)
+ goto out;
- used = min_t(unsigned long, ctx->used, msg_data_left(msg));
-
- used = af_alg_make_sg(&ctx->rsgl, &msg->msg_iter, used);
- err = used;
- if (err < 0)
- goto unlock;
-
- if (ctx->more || used < ctx->used)
- used -= used % bs;
-
- err = -EINVAL;
- if (!used)
- goto free;
-
- sgl = list_first_entry(&ctx->tsgl,
- struct skcipher_sg_list, list);
- sg = sgl->sg;
-
- while (!sg->length)
- sg++;
-
- skcipher_request_set_crypt(&ctx->req, sg, ctx->rsgl.sg, used,
- ctx->iv);
-
- err = af_alg_wait_for_completion(
- ctx->enc ?
- crypto_skcipher_encrypt(&ctx->req) :
- crypto_skcipher_decrypt(&ctx->req),
- &ctx->completion);
-
-free:
- af_alg_free_sg(&ctx->rsgl);
-
- if (err)
- goto unlock;
-
- copied += used;
- skcipher_pull_sgl(sk, used, 1);
- iov_iter_advance(&msg->msg_iter, used);
+ ret += err;
}
- err = 0;
-
-unlock:
+out:
skcipher_wmem_wakeup(sk);
release_sock(sk);
-
- return copied ?: err;
-}
-
-static int skcipher_recvmsg(struct socket *sock, struct msghdr *msg,
- size_t ignored, int flags)
-{
- return (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) ?
- skcipher_recvmsg_async(sock, msg, flags) :
- skcipher_recvmsg_sync(sock, msg, flags);
+ return ret;
}
static unsigned int skcipher_poll(struct file *file, struct socket *sock,
@@ -724,10 +749,9 @@ static unsigned int skcipher_poll(struct file *file, struct socket *sock,
struct sock *sk = sock->sk;
struct alg_sock *ask = alg_sk(sk);
struct skcipher_ctx *ctx = ask->private;
- unsigned int mask;
+ unsigned int mask = 0;
sock_poll_wait(file, sk_sleep(sk), wait);
- mask = 0;
if (ctx->used)
mask |= POLLIN | POLLRDNORM;
@@ -895,26 +919,20 @@ static int skcipher_setkey(void *private, const u8 *key, unsigned int keylen)
return err;
}
-static void skcipher_wait(struct sock *sk)
-{
- struct alg_sock *ask = alg_sk(sk);
- struct skcipher_ctx *ctx = ask->private;
- int ctr = 0;
-
- while (atomic_read(&ctx->inflight) && ctr++ < 100)
- msleep(100);
-}
-
static void skcipher_sock_destruct(struct sock *sk)
{
struct alg_sock *ask = alg_sk(sk);
struct skcipher_ctx *ctx = ask->private;
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(&ctx->req);
+ struct sock *psk = ask->parent;
+ struct alg_sock *pask = alg_sk(psk);
+ struct skcipher_tfm *skc = pask->private;
+ struct crypto_skcipher *tfm = skc->skcipher;
- if (atomic_read(&ctx->inflight))
- skcipher_wait(sk);
+ /* Suspend caller if AIO operations are in flight. */
+ wait_event_interruptible(skcipher_aio_finish_wait,
+ (ctx->inflight == 0));
- skcipher_free_sgl(sk);
+ skcipher_pull_tsgl(sk, ctx->used, NULL);
sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm));
sock_kfree_s(sk, ctx, ctx->len);
af_alg_release_parent(sk);
@@ -926,7 +944,7 @@ static int skcipher_accept_parent_nokey(void *private, struct sock *sk)
struct alg_sock *ask = alg_sk(sk);
struct skcipher_tfm *tfm = private;
struct crypto_skcipher *skcipher = tfm->skcipher;
- unsigned int len = sizeof(*ctx) + crypto_skcipher_reqsize(skcipher);
+ unsigned int len = sizeof(*ctx);
ctx = sock_kmalloc(sk, len, GFP_KERNEL);
if (!ctx)
@@ -941,22 +959,18 @@ static int skcipher_accept_parent_nokey(void *private, struct sock *sk)
memset(ctx->iv, 0, crypto_skcipher_ivsize(skcipher));
- INIT_LIST_HEAD(&ctx->tsgl);
+ INIT_LIST_HEAD(&ctx->tsgl_list);
ctx->len = len;
ctx->used = 0;
+ ctx->rcvused = 0;
+ ctx->inflight = 0;
ctx->more = 0;
ctx->merge = 0;
ctx->enc = 0;
- atomic_set(&ctx->inflight, 0);
af_alg_init_completion(&ctx->completion);
ask->private = ctx;
- skcipher_request_set_tfm(&ctx->req, skcipher);
- skcipher_request_set_callback(&ctx->req, CRYPTO_TFM_REQ_MAY_SLEEP |
- CRYPTO_TFM_REQ_MAY_BACKLOG,
- af_alg_complete, &ctx->completion);
-
sk->sk_destruct = skcipher_sock_destruct;
return 0;
--
2.9.3
^ permalink raw reply related
* [PATCH v5 0/4] gf128mul refactoring
From: Ondrej Mosnacek @ 2017-04-02 19:19 UTC (permalink / raw)
To: Herbert Xu
Cc: Ondrej Mosnacek, David S. Miller, linux-crypto, Eric Biggers,
Jeffrey Walton, Milan Broz
This patchset contains the following gf128mul-related changes:
1. The gf128mul_x_* functions are moved to gf128mul.h for performance reasons.
2. The gf128mul_x_ble function is fixed to use the correct block type.
3. The le128_gf128mul_x_ble function from glue_helper.h is removed and its
usages replaced with gf128mul_x_ble calls.
4. The now obsolete dependency of CRYPTO_XTS on CRYPTO_GF128MUL is removed.
v4 -> v5: added the other three patches
v3 -> v4: a faster version of gf128mul_x_lle
v2 -> v3: constant-time implementation
v1 -> v2: move all _x_ functions to the header, not just gf128mul_x_ble
Ondrej Mosnacek (4):
crypto: gf128mul - define gf128mul_x_* in gf128mul.h
crypto: gf128mul - switch gf128mul_x_ble to le128
crypto: glue_helper - remove the le128_gf128mul_x_ble function
crypto: xts - drop gf128mul dependency
arch/x86/crypto/camellia_glue.c | 4 +--
arch/x86/crypto/glue_helper.c | 3 +-
arch/x86/crypto/serpent_sse2_glue.c | 4 +--
arch/x86/crypto/twofish_glue_3way.c | 4 +--
arch/x86/include/asm/crypto/glue_helper.h | 10 ------
crypto/Kconfig | 1 -
crypto/gf128mul.c | 33 +------------------
crypto/xts.c | 38 ++++++++++-----------
include/crypto/gf128mul.h | 55 +++++++++++++++++++++++++++++--
include/crypto/xts.h | 2 +-
10 files changed, 82 insertions(+), 72 deletions(-)
--
2.9.3
^ permalink raw reply
* [PATCH v5 1/4] crypto: gf128mul - define gf128mul_x_* in gf128mul.h
From: Ondrej Mosnacek @ 2017-04-02 19:19 UTC (permalink / raw)
To: Herbert Xu
Cc: Ondrej Mosnacek, David S. Miller, linux-crypto, Eric Biggers,
Jeffrey Walton, Milan Broz
In-Reply-To: <20170402191916.9309-1-omosnacek@gmail.com>
The gf128mul_x_ble function is currently defined in gf128mul.c, because
it depends on the gf128mul_table_be multiplication table.
However, since the function is very small and only uses two values from
the table, it is better for it to be defined as inline function in
gf128mul.h. That way, the function can be inlined by the compiler for
better performance.
For consistency, the other gf128mul_x_* functions are also moved to the
header file. In addition, the code is rewritten to be constant-time.
After this change, the speed of the generic 'xts(aes)' implementation
increased from ~225 MiB/s to ~235 MiB/s (measured using 'cryptsetup
benchmark -c aes-xts-plain64' on an Intel system with CRYPTO_AES_X86_64
and CRYPTO_AES_NI_INTEL disabled).
Signed-off-by: Ondrej Mosnacek <omosnacek@gmail.com>
Cc: Eric Biggers <ebiggers@google.com>
---
crypto/gf128mul.c | 33 +---------------------------
include/crypto/gf128mul.h | 55 +++++++++++++++++++++++++++++++++++++++++++++--
2 files changed, 54 insertions(+), 34 deletions(-)
diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c
index 04facc0..dc01212 100644
--- a/crypto/gf128mul.c
+++ b/crypto/gf128mul.c
@@ -130,43 +130,12 @@ static const u16 gf128mul_table_le[256] = gf128mul_dat(xda_le);
static const u16 gf128mul_table_be[256] = gf128mul_dat(xda_be);
/*
- * The following functions multiply a field element by x or by x^8 in
+ * The following functions multiply a field element by x^8 in
* the polynomial field representation. They use 64-bit word operations
* to gain speed but compensate for machine endianness and hence work
* correctly on both styles of machine.
*/
-static void gf128mul_x_lle(be128 *r, const be128 *x)
-{
- u64 a = be64_to_cpu(x->a);
- u64 b = be64_to_cpu(x->b);
- u64 _tt = gf128mul_table_le[(b << 7) & 0xff];
-
- r->b = cpu_to_be64((b >> 1) | (a << 63));
- r->a = cpu_to_be64((a >> 1) ^ (_tt << 48));
-}
-
-static void gf128mul_x_bbe(be128 *r, const be128 *x)
-{
- u64 a = be64_to_cpu(x->a);
- u64 b = be64_to_cpu(x->b);
- u64 _tt = gf128mul_table_be[a >> 63];
-
- r->a = cpu_to_be64((a << 1) | (b >> 63));
- r->b = cpu_to_be64((b << 1) ^ _tt);
-}
-
-void gf128mul_x_ble(be128 *r, const be128 *x)
-{
- u64 a = le64_to_cpu(x->a);
- u64 b = le64_to_cpu(x->b);
- u64 _tt = gf128mul_table_be[b >> 63];
-
- r->a = cpu_to_le64((a << 1) ^ _tt);
- r->b = cpu_to_le64((b << 1) | (a >> 63));
-}
-EXPORT_SYMBOL(gf128mul_x_ble);
-
static void gf128mul_x8_lle(be128 *x)
{
u64 a = be64_to_cpu(x->a);
diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h
index 0bc9b5f..35ced9d 100644
--- a/include/crypto/gf128mul.h
+++ b/include/crypto/gf128mul.h
@@ -49,6 +49,7 @@
#ifndef _CRYPTO_GF128MUL_H
#define _CRYPTO_GF128MUL_H
+#include <asm/byteorder.h>
#include <crypto/b128ops.h>
#include <linux/slab.h>
@@ -163,8 +164,58 @@ void gf128mul_lle(be128 *a, const be128 *b);
void gf128mul_bbe(be128 *a, const be128 *b);
-/* multiply by x in ble format, needed by XTS */
-void gf128mul_x_ble(be128 *a, const be128 *b);
+/*
+ * The following functions multiply a field element by x in
+ * the polynomial field representation. They use 64-bit word operations
+ * to gain speed but compensate for machine endianness and hence work
+ * correctly on both styles of machine.
+ *
+ * They are defined here for performance.
+ */
+
+static inline u64 gf128mul_mask_from_bit(u64 x, int which)
+{
+ /* a constant-time version of 'x & ((u64)1 << which) ? (u64)-1 : 0' */
+ return ((s64)(x << (63 - which)) >> 63);
+}
+
+static inline void gf128mul_x_lle(be128 *r, const be128 *x)
+{
+ u64 a = be64_to_cpu(x->a);
+ u64 b = be64_to_cpu(x->b);
+
+ /* equivalent to gf128mul_table_le[(b << 7) & 0xff] << 48
+ * (see crypto/gf128mul.c): */
+ u64 _tt = gf128mul_mask_from_bit(b, 0) & ((u64)0xe1 << 56);
+
+ r->b = cpu_to_be64((b >> 1) | (a << 63));
+ r->a = cpu_to_be64((a >> 1) ^ _tt);
+}
+
+static inline void gf128mul_x_bbe(be128 *r, const be128 *x)
+{
+ u64 a = be64_to_cpu(x->a);
+ u64 b = be64_to_cpu(x->b);
+
+ /* equivalent to gf128mul_table_be[a >> 63] (see crypto/gf128mul.c): */
+ u64 _tt = gf128mul_mask_from_bit(a, 63) & 0x87;
+
+ r->a = cpu_to_be64((a << 1) | (b >> 63));
+ r->b = cpu_to_be64((b << 1) ^ _tt);
+}
+
+/* needed by XTS */
+static inline void gf128mul_x_ble(be128 *r, const be128 *x)
+{
+ u64 a = le64_to_cpu(x->a);
+ u64 b = le64_to_cpu(x->b);
+
+ /* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */
+ u64 _tt = gf128mul_mask_from_bit(b, 63) & 0x87;
+
+ r->a = cpu_to_le64((a << 1) ^ _tt);
+ r->b = cpu_to_le64((b << 1) | (a >> 63));
+}
/* 4k table optimization */
--
2.9.3
^ permalink raw reply related
* [PATCH v5 2/4] crypto: gf128mul - switch gf128mul_x_ble to le128
From: Ondrej Mosnacek @ 2017-04-02 19:19 UTC (permalink / raw)
To: Herbert Xu
Cc: Ondrej Mosnacek, David S. Miller, linux-crypto, Eric Biggers,
Milan Broz
In-Reply-To: <20170402191916.9309-1-omosnacek@gmail.com>
Currently, gf128mul_x_ble works with pointers to be128, even though it
actually interprets the words as little-endian. Consequently, it uses
cpu_to_le64/le64_to_cpu on fields of type __be64, which is incorrect.
This patch fixes that by changing the function to accept pointers to
le128 and updating all users accordingly.
Signed-off-by: Ondrej Mosnacek <omosnacek@gmail.com>
---
arch/x86/crypto/camellia_glue.c | 4 ++--
arch/x86/crypto/serpent_sse2_glue.c | 4 ++--
arch/x86/crypto/twofish_glue_3way.c | 4 ++--
crypto/xts.c | 38 ++++++++++++++++++-------------------
include/crypto/gf128mul.h | 8 ++++----
include/crypto/xts.h | 2 +-
6 files changed, 30 insertions(+), 30 deletions(-)
diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c
index aa76cad..af4840a 100644
--- a/arch/x86/crypto/camellia_glue.c
+++ b/arch/x86/crypto/camellia_glue.c
@@ -1522,7 +1522,7 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- be128 buf[2 * 4];
+ le128 buf[2 * 4];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
@@ -1540,7 +1540,7 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- be128 buf[2 * 4];
+ le128 buf[2 * 4];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c
index 644f97a..ac0e831 100644
--- a/arch/x86/crypto/serpent_sse2_glue.c
+++ b/arch/x86/crypto/serpent_sse2_glue.c
@@ -328,7 +328,7 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- be128 buf[SERPENT_PARALLEL_BLOCKS];
+ le128 buf[SERPENT_PARALLEL_BLOCKS];
struct crypt_priv crypt_ctx = {
.ctx = &ctx->crypt_ctx,
.fpu_enabled = false,
@@ -355,7 +355,7 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- be128 buf[SERPENT_PARALLEL_BLOCKS];
+ le128 buf[SERPENT_PARALLEL_BLOCKS];
struct crypt_priv crypt_ctx = {
.ctx = &ctx->crypt_ctx,
.fpu_enabled = false,
diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c
index 2ebb5e9..243e90a 100644
--- a/arch/x86/crypto/twofish_glue_3way.c
+++ b/arch/x86/crypto/twofish_glue_3way.c
@@ -296,7 +296,7 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- be128 buf[3];
+ le128 buf[3];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
@@ -314,7 +314,7 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- be128 buf[3];
+ le128 buf[3];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
diff --git a/crypto/xts.c b/crypto/xts.c
index baeb34d..bd5065c 100644
--- a/crypto/xts.c
+++ b/crypto/xts.c
@@ -39,11 +39,11 @@ struct xts_instance_ctx {
};
struct rctx {
- be128 buf[XTS_BUFFER_SIZE / sizeof(be128)];
+ le128 buf[XTS_BUFFER_SIZE / sizeof(le128)];
- be128 t;
+ le128 t;
- be128 *ext;
+ le128 *ext;
struct scatterlist srcbuf[2];
struct scatterlist dstbuf[2];
@@ -99,7 +99,7 @@ static int setkey(struct crypto_skcipher *parent, const u8 *key,
static int post_crypt(struct skcipher_request *req)
{
struct rctx *rctx = skcipher_request_ctx(req);
- be128 *buf = rctx->ext ?: rctx->buf;
+ le128 *buf = rctx->ext ?: rctx->buf;
struct skcipher_request *subreq;
const int bs = XTS_BLOCK_SIZE;
struct skcipher_walk w;
@@ -112,12 +112,12 @@ static int post_crypt(struct skcipher_request *req)
while (w.nbytes) {
unsigned int avail = w.nbytes;
- be128 *wdst;
+ le128 *wdst;
wdst = w.dst.virt.addr;
do {
- be128_xor(wdst, buf++, wdst);
+ le128_xor(wdst, buf++, wdst);
wdst++;
} while ((avail -= bs) >= bs);
@@ -150,7 +150,7 @@ static int post_crypt(struct skcipher_request *req)
static int pre_crypt(struct skcipher_request *req)
{
struct rctx *rctx = skcipher_request_ctx(req);
- be128 *buf = rctx->ext ?: rctx->buf;
+ le128 *buf = rctx->ext ?: rctx->buf;
struct skcipher_request *subreq;
const int bs = XTS_BLOCK_SIZE;
struct skcipher_walk w;
@@ -174,15 +174,15 @@ static int pre_crypt(struct skcipher_request *req)
while (w.nbytes) {
unsigned int avail = w.nbytes;
- be128 *wsrc;
- be128 *wdst;
+ le128 *wsrc;
+ le128 *wdst;
wsrc = w.src.virt.addr;
wdst = w.dst.virt.addr;
do {
*buf++ = rctx->t;
- be128_xor(wdst++, &rctx->t, wsrc++);
+ le128_xor(wdst++, &rctx->t, wsrc++);
gf128mul_x_ble(&rctx->t, &rctx->t);
} while ((avail -= bs) >= bs);
@@ -350,8 +350,8 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
const unsigned int max_blks = req->tbuflen / bsize;
struct blkcipher_walk walk;
unsigned int nblocks;
- be128 *src, *dst, *t;
- be128 *t_buf = req->tbuf;
+ le128 *src, *dst, *t;
+ le128 *t_buf = req->tbuf;
int err, i;
BUG_ON(max_blks < 1);
@@ -364,8 +364,8 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
return err;
nblocks = min(nbytes / bsize, max_blks);
- src = (be128 *)walk.src.virt.addr;
- dst = (be128 *)walk.dst.virt.addr;
+ src = (le128 *)walk.src.virt.addr;
+ dst = (le128 *)walk.dst.virt.addr;
/* calculate first value of T */
req->tweak_fn(req->tweak_ctx, (u8 *)&t_buf[0], walk.iv);
@@ -381,7 +381,7 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
t = &t_buf[i];
/* PP <- T xor P */
- be128_xor(dst + i, t, src + i);
+ le128_xor(dst + i, t, src + i);
}
/* CC <- E(Key2,PP) */
@@ -390,7 +390,7 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
/* C <- T xor CC */
for (i = 0; i < nblocks; i++)
- be128_xor(dst + i, dst + i, &t_buf[i]);
+ le128_xor(dst + i, dst + i, &t_buf[i]);
src += nblocks;
dst += nblocks;
@@ -398,7 +398,7 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
nblocks = min(nbytes / bsize, max_blks);
} while (nblocks > 0);
- *(be128 *)walk.iv = *t;
+ *(le128 *)walk.iv = *t;
err = blkcipher_walk_done(desc, &walk, nbytes);
nbytes = walk.nbytes;
@@ -406,8 +406,8 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
break;
nblocks = min(nbytes / bsize, max_blks);
- src = (be128 *)walk.src.virt.addr;
- dst = (be128 *)walk.dst.virt.addr;
+ src = (le128 *)walk.src.virt.addr;
+ dst = (le128 *)walk.dst.virt.addr;
}
return err;
diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h
index 35ced9d..0977fb1 100644
--- a/include/crypto/gf128mul.h
+++ b/include/crypto/gf128mul.h
@@ -205,16 +205,16 @@ static inline void gf128mul_x_bbe(be128 *r, const be128 *x)
}
/* needed by XTS */
-static inline void gf128mul_x_ble(be128 *r, const be128 *x)
+static inline void gf128mul_x_ble(le128 *r, const le128 *x)
{
u64 a = le64_to_cpu(x->a);
u64 b = le64_to_cpu(x->b);
/* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */
- u64 _tt = gf128mul_mask_from_bit(b, 63) & 0x87;
+ u64 _tt = gf128mul_mask_from_bit(a, 63) & 0x87;
- r->a = cpu_to_le64((a << 1) ^ _tt);
- r->b = cpu_to_le64((b << 1) | (a >> 63));
+ r->a = cpu_to_le64((a << 1) | (b >> 63));
+ r->b = cpu_to_le64((b << 1) ^ _tt);
}
/* 4k table optimization */
diff --git a/include/crypto/xts.h b/include/crypto/xts.h
index 77b6306..c0bde30 100644
--- a/include/crypto/xts.h
+++ b/include/crypto/xts.h
@@ -11,7 +11,7 @@ struct blkcipher_desc;
#define XTS_BLOCK_SIZE 16
struct xts_crypt_req {
- be128 *tbuf;
+ le128 *tbuf;
unsigned int tbuflen;
void *tweak_ctx;
--
2.9.3
^ permalink raw reply related
* [PATCH v5 3/4] crypto: glue_helper - remove the le128_gf128mul_x_ble function
From: Ondrej Mosnacek @ 2017-04-02 19:19 UTC (permalink / raw)
To: Herbert Xu
Cc: Ondrej Mosnacek, David S. Miller, linux-crypto, Eric Biggers,
Milan Broz
In-Reply-To: <20170402191916.9309-1-omosnacek@gmail.com>
The le128_gf128mul_x_ble function in glue_helper.h is now obsolete and
can be replaced with the gf128mul_x_ble function from gf128mul.h.
Signed-off-by: Ondrej Mosnacek <omosnacek@gmail.com>
---
arch/x86/crypto/glue_helper.c | 3 ++-
arch/x86/include/asm/crypto/glue_helper.h | 10 ----------
2 files changed, 2 insertions(+), 11 deletions(-)
diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c
index 260a060..24ac9fa 100644
--- a/arch/x86/crypto/glue_helper.c
+++ b/arch/x86/crypto/glue_helper.c
@@ -27,6 +27,7 @@
#include <linux/module.h>
#include <crypto/b128ops.h>
+#include <crypto/gf128mul.h>
#include <crypto/internal/skcipher.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
@@ -457,7 +458,7 @@ void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, le128 *iv,
le128 ivblk = *iv;
/* generate next IV */
- le128_gf128mul_x_ble(iv, &ivblk);
+ gf128mul_x_ble(iv, &ivblk);
/* CC <- T xor C */
u128_xor(dst, src, (u128 *)&ivblk);
diff --git a/arch/x86/include/asm/crypto/glue_helper.h b/arch/x86/include/asm/crypto/glue_helper.h
index 29e53ea..ed8b66d 100644
--- a/arch/x86/include/asm/crypto/glue_helper.h
+++ b/arch/x86/include/asm/crypto/glue_helper.h
@@ -125,16 +125,6 @@ static inline void le128_inc(le128 *i)
i->b = cpu_to_le64(b);
}
-static inline void le128_gf128mul_x_ble(le128 *dst, const le128 *src)
-{
- u64 a = le64_to_cpu(src->a);
- u64 b = le64_to_cpu(src->b);
- u64 _tt = ((s64)a >> 63) & 0x87;
-
- dst->a = cpu_to_le64((a << 1) ^ (b >> 63));
- dst->b = cpu_to_le64((b << 1) ^ _tt);
-}
-
extern int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
struct blkcipher_desc *desc,
struct scatterlist *dst,
--
2.9.3
^ permalink raw reply related
* [PATCH v5 4/4] crypto: xts - drop gf128mul dependency
From: Ondrej Mosnacek @ 2017-04-02 19:19 UTC (permalink / raw)
To: Herbert Xu
Cc: Ondrej Mosnacek, David S. Miller, linux-crypto, Eric Biggers,
Milan Broz
In-Reply-To: <20170402191916.9309-1-omosnacek@gmail.com>
Since the gf128mul_x_ble function used by xts.c is now defined inline
in the header file, the XTS module no longer depends on gf128mul.
Therefore, the 'select CRYPTO_GF128MUL' line can be safely removed.
Signed-off-by: Ondrej Mosnacek <omosnacek@gmail.com>
---
crypto/Kconfig | 1 -
1 file changed, 1 deletion(-)
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 6854c1f..aac4bc9 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -374,7 +374,6 @@ config CRYPTO_XTS
tristate "XTS support"
select CRYPTO_BLKCIPHER
select CRYPTO_MANAGER
- select CRYPTO_GF128MUL
select CRYPTO_ECB
help
XTS: IEEE1619/D16 narrow block cipher use with aes-xts-plain,
--
2.9.3
^ permalink raw reply related
* Re: [PATCH 4/5] crypto/nx: Add P9 NX support for 842 compression engine.
From: Stewart Smith @ 2017-04-03 1:37 UTC (permalink / raw)
To: Haren Myneni, herbert, mpe, ddstreet
Cc: mikey, suka, linux-crypto, linuxppc-dev
In-Reply-To: <1491066107.29552.29.camel@hbabu-laptop>
Haren Myneni <haren@linux.vnet.ibm.com> writes:
> @@ -656,13 +953,21 @@ static __init int nx842_powernv_init(void)
> BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT);
> BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
>
> - for_each_compatible_node(dn, NULL, "ibm,power-nx")
> - nx842_powernv_probe(dn);
> + if (is_vas_available()) {
> + for_each_compatible_node(dn, NULL, "ibm,xscom")
> + nx842_powernv_probe_vas(dn);
I'm not keen on how the device bindings work, instead, I think firmware
should provide a 'ibm,vas' compatible node, rather than simply searching
through all the ibm,xscom nodes.
XSCOMs aren't something that Linux should really know about, it's a
debug interface, and one we use through PRD to do PRD-things, XSCOMs
aren't part of the architecture.
--
Stewart Smith
OPAL Architect, IBM.
^ permalink raw reply
* Re: algif for compression?
From: abed mohammad kamaluddin @ 2017-04-03 6:10 UTC (permalink / raw)
To: Herbert Xu; +Cc: linux-crypto
In-Reply-To: <20161210081014.GA32746@gondor.apana.org.au>
Hi Herbert,
We have implemented algif acomp interface for user space applications
to utilize the kernel compression framework and the hardware drivers
using it and have been testing it using userspace zlib library.
However, what we find lacking in the existing acomp implementation is
the ability to pass context data between the applications and the
drivers using the acomp interface. Currently the interface only allows
src/dest data and a flag argument with each request. There are two
context pointers, one in acomp_req and another in crypto_tfm but they
are for internal use and not available to applications through the
api's. Would it be acceptable to add fields that need to be
communicated b/w the driver and applications like history,
csum/adler32, EOF, stream ctx to acomp_req.
Or is there any other way, which I may have missed, through which we
can pass ctx data between applications and drivers while using the
kernel compression framework?
Thanks,
Abed
(Cavium Inc)
On Sat, Dec 10, 2016 at 1:40 PM, Herbert Xu <herbert@gondor.apana.org.au> wrote:
> abed mohammad kamaluddin <abedamu@gmail.com> wrote:
>>
>> We are also looking for user-space access to the kernel crypto layer
>> compression algorithms. I have gone through AF_ALG but couldn’t find
>> support for compression ops through it. This is required for our
>> hardware zip engines whose driver hooks up to the crypto layer.
>>
>> I was going through the lists and stumbled across this thread. Has
>> there been any updates or efforts put up in this direction since this
>> thread is a few years old. If not, are there any alternate
>> implementations that allow user-space access to compression? I have
>> gone through cryptodev and see the same limitation there.
>>
>> Would appreciate any pointers in this regard.
>
> The compression interface is currently in a state of flux. We
> should make it settle down first before exporting it to user-space.
>
> For a start it would be good to actually switch IPsec/IPcomp over
> to the new compression interface.
>
> Cheers,
> --
> Email: Herbert Xu <herbert@gondor.apana.org.au>
> Home Page: http://gondor.apana.org.au/~herbert/
> PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply
* Re: algif for compression?
From: Stephan Müller @ 2017-04-03 7:26 UTC (permalink / raw)
To: abed mohammad kamaluddin; +Cc: Herbert Xu, linux-crypto
In-Reply-To: <CACVarEQ14ZF9LGDkaNYtX3GRD5GOH62NBiPtC4z=i6RY4MtFDA@mail.gmail.com>
Am Montag, 3. April 2017, 08:10:19 CEST schrieb abed mohammad kamaluddin:
Hi abed,
> Hi Herbert,
>
> We have implemented algif acomp interface for user space applications
> to utilize the kernel compression framework and the hardware drivers
> using it and have been testing it using userspace zlib library.
>
> However, what we find lacking in the existing acomp implementation is
> the ability to pass context data between the applications and the
> drivers using the acomp interface. Currently the interface only allows
> src/dest data and a flag argument with each request. There are two
> context pointers, one in acomp_req and another in crypto_tfm but they
> are for internal use and not available to applications through the
> api's. Would it be acceptable to add fields that need to be
> communicated b/w the driver and applications like history,
> csum/adler32, EOF, stream ctx to acomp_req.
>
> Or is there any other way, which I may have missed, through which we
> can pass ctx data between applications and drivers while using the
> kernel compression framework?
If you follow the AF_ALG implementations that are currently in the kernel, a
calling application receives two file descriptors. The first file descriptor
is the reference to a tfm, the second is the reference to a particular
request.
Wouldn't those file descriptors be the reference you are looking for?
Ciao
Stephan
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox