* Re: [PATCH 0/6] Add support for ECDSA algorithm
From: Nitin Kumbhar @ 2017-02-03 11:16 UTC (permalink / raw)
To: Herbert Xu; +Cc: nkumbhar, davem, linux-crypto
In-Reply-To: <20170202135721.GA5289@gondor.apana.org.au>
Hello Herbert,
On 2/2/2017 7:27 PM, Herbert Xu wrote:
> On Thu, Jan 26, 2017 at 11:30:04AM +0530, Nitin Kumbhar wrote:
>>
>> This ECDSA implementation is analogous to the RSA kernel implementation for
>> signature generation / verification. It extends ECC family of algorithms
>> like ECDH to support signature verification using akcipher. This will be
>> used in a way similar to RSA.
>
> Yes but RSA had an in-kernel user in the form of module signature
> verification. We don't add algorithms to the kernel without
> actual users. So this patch-set needs to come with an actual
> in-kernel user of ECDSA.
If this is about ECC in kernel then ECDSA extends ECDH which is from ECC
and already part of kernel.
If none of the above are sufficient for now then let these patches come
along with a consumer or let kernel add ECDSA based on akcipher as part
of a crypto engine.
Nonetheless, how about rest of the vli and ECC patches?
> Thanks,
>
Regards,
- Nitin
-----------------------------------------------------------------------------------
This email message is for the sole use of the intended recipient(s) and may contain
confidential information. Any unauthorized review, use, disclosure or distribution
is prohibited. If you are not the intended recipient, please contact the sender by
reply email and destroy all copies of the original message.
-----------------------------------------------------------------------------------
^ permalink raw reply
* [PATCH v2 6/6] crypto: tcrypt: add ECDSA test modes
From: Nitin Kumbhar @ 2017-02-03 11:12 UTC (permalink / raw)
To: herbert, davem; +Cc: linux-crypto, Nitin Kumbhar
In-Reply-To: <1486120375-13070-1-git-send-email-nkumbhar@nvidia.com>
Update tcrypt module to include a new ECDSA test
modes. It includes:
tcrypt.ko mode=560 for ECDSA sign/verify validation.
tcrypt.ko mode=561 for ECDSA sign/verify op perf in cycles.
tcrypt.ko mode=561 sec=<seconds> for number of ECDSA sign
verify ops in given time.
Signed-off-by: Nitin Kumbhar <nkumbhar@nvidia.com>
---
crypto/tcrypt.c | 250 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
crypto/tcrypt.h | 122 +++++++++++++++++++++++++++
2 files changed, 368 insertions(+), 4 deletions(-)
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index 9a11f3c2bf98..1723c6ef5b4f 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -7,6 +7,7 @@
* Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
* Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
* Copyright (c) 2007 Nokia Siemens Networks
+ * Copyright (c) 2017 NVIDIA Corporation
*
* Updated RFC4106 AES-GCM testing.
* Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
@@ -27,6 +28,7 @@
#include <crypto/aead.h>
#include <crypto/hash.h>
#include <crypto/skcipher.h>
+#include <crypto/akcipher.h>
#include <linux/err.h>
#include <linux/fips.h>
#include <linux/init.h>
@@ -46,10 +48,12 @@
#define TVMEMSIZE 4
/*
-* Used by test_cipher_speed()
-*/
-#define ENCRYPT 1
-#define DECRYPT 0
+ * Used by test_cipher_speed()
+ */
+#define DECRYPT 0
+#define ENCRYPT 1
+#define SIGN 2
+#define VERIFY 3
#define MAX_DIGEST_SIZE 64
@@ -996,6 +1000,223 @@ static void test_cipher_speed(const char *algo, int enc, unsigned int secs,
false);
}
+static inline int do_one_akcipher_op(struct akcipher_request *r, int ret)
+{
+ if (ret == -EINPROGRESS || ret == -EBUSY) {
+ struct tcrypt_result *tr = r->base.data;
+
+ wait_for_completion(&tr->completion);
+ reinit_completion(&tr->completion);
+ ret = tr->err;
+ }
+ return ret;
+}
+
+static int test_akcipher_jiffies(struct akcipher_request *r, int op, int secs)
+{
+ unsigned long start, end;
+ int count, ret;
+
+ for (start = jiffies, end = start + secs * HZ, count = 0;
+ time_before(jiffies, end); count++) {
+
+ switch (op) {
+ case SIGN:
+ ret = do_one_akcipher_op(r, crypto_akcipher_sign(r));
+ break;
+ case VERIFY:
+ ret = do_one_akcipher_op(r, crypto_akcipher_verify(r));
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+ if (ret)
+ return ret;
+ }
+
+ pr_info("%d operations in %d seconds\n", count, secs);
+ return 0;
+}
+
+static int test_akcipher_cycles(struct akcipher_request *r, int op)
+{
+ unsigned long cycles = 0;
+ int ret = 0;
+ int i;
+
+ /* Warm-up run. */
+ for (i = 0; i < 4; i++) {
+ switch (op) {
+ case SIGN:
+ ret = do_one_akcipher_op(r, crypto_akcipher_sign(r));
+ break;
+ case VERIFY:
+ ret = do_one_akcipher_op(r, crypto_akcipher_verify(r));
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+ if (ret)
+ goto out;
+ }
+
+ /* The real thing. */
+ for (i = 0; i < 8; i++) {
+ cycles_t start, end;
+
+ start = get_cycles();
+ switch (op) {
+ case SIGN:
+ ret = do_one_akcipher_op(r, crypto_akcipher_sign(r));
+ break;
+ case VERIFY:
+ ret = do_one_akcipher_op(r, crypto_akcipher_verify(r));
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+ end = get_cycles();
+
+ if (ret)
+ goto out;
+
+ cycles += end - start;
+ }
+out:
+ if (ret == 0)
+ pr_info("1 operation in %lu cycles\n", (cycles + 4) / 8);
+
+ return ret;
+}
+
+static void test_akcipher_speed(const char *algo, int op, unsigned int secs,
+ struct akcipher_speed_template *template,
+ unsigned int tcount, u8 *keysize)
+{
+ unsigned int ret, i, j;
+ struct tcrypt_result tresult;
+ const char *key;
+ struct akcipher_request *req;
+ struct crypto_akcipher *tfm;
+ unsigned int m_size = 0;
+ unsigned int nbytes = 0;
+ const char *o;
+
+ if (op == SIGN)
+ o = "sign";
+ else if (op == VERIFY)
+ o = "verify";
+ else
+ return;
+
+ tfm = crypto_alloc_akcipher(algo, 0, 0);
+ if (IS_ERR(tfm)) {
+ pr_err("failed to load transform for %s: %ld\n", algo,
+ PTR_ERR(tfm));
+ return;
+ }
+
+ req = akcipher_request_alloc(tfm, GFP_KERNEL);
+ if (!req) {
+ pr_err("tcrypt: akcipher: Failed to allocate request for %s\n",
+ algo);
+ goto out;
+ }
+
+ init_completion(&tresult.completion);
+ akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+ tcrypt_complete, &tresult);
+
+ i = 0;
+ do {
+ struct scatterlist sg[TVMEMSIZE];
+
+ memset(tvmem[0], 0xff, PAGE_SIZE);
+
+ /* set key */
+ key = tvmem[0];
+ for (j = 0; j < tcount; j++) {
+ if (template[j].key_len == *keysize) {
+ key = template[j].key;
+ break;
+ }
+ }
+
+ ret = crypto_akcipher_set_pub_key(tfm, key, *keysize);
+ if (ret) {
+ pr_err("set_pub_key() failed\n");
+ goto out_free_req;
+ }
+
+ ret = crypto_akcipher_set_priv_key(tfm, key, *keysize);
+ if (ret) {
+ pr_err("set_priv_key() failed\n");
+ goto out_free_req;
+ }
+
+ /* set up src/dst buffs */
+ sg_init_table(sg, TVMEMSIZE);
+ if (op == SIGN) {
+ m_size = template[j].m_size;
+ nbytes = template[j].c_size / 3;
+
+ memcpy(tvmem[0], template[j].m, m_size);
+
+ sg_set_buf(&sg[0], tvmem[0], m_size);
+ akcipher_request_set_crypt(req, sg, sg,
+ m_size, PAGE_SIZE);
+ } else if (op == VERIFY) {
+ m_size = template[j].m_size;
+ nbytes = template[j].c_size / 3;
+
+ memcpy(tvmem[0], template[j].m, m_size);
+ memcpy(tvmem[1], (u8 *)(template[j].c) + nbytes,
+ nbytes);
+ memcpy(tvmem[2], (u8 *)(template[j].c) + 2 * nbytes,
+ nbytes);
+
+ sg_set_buf(&sg[0], tvmem[0], m_size);
+ sg_set_buf(&sg[1], tvmem[1], nbytes);
+ sg_set_buf(&sg[2], tvmem[2], nbytes);
+
+ akcipher_request_set_crypt(req, sg, sg,
+ m_size + 2 * nbytes,
+ PAGE_SIZE);
+ } else {
+ pr_err("invalid op\n");
+ ret = -EINVAL;
+ goto out_free_req;
+ }
+
+
+ pr_info("\ntesting speed of %s (%s) %s with keysize %d\n",
+ algo, get_driver_name(crypto_akcipher, tfm), o,
+ nbytes * 8);
+
+ if (secs)
+ ret = test_akcipher_jiffies(req, op, secs);
+ else
+ ret = test_akcipher_cycles(req, op);
+
+ if (ret) {
+ pr_err("%s() failed\n", o);
+ break;
+ }
+
+ i++;
+ keysize++;
+
+ } while (*keysize);
+
+out_free_req:
+ akcipher_request_free(req);
+out:
+ crypto_free_akcipher(tfm);
+}
+
static void test_available(void)
{
char **name = check;
@@ -2039,6 +2260,27 @@ static int do_test(const char *alg, u32 type, u32 mask, int m)
speed_template_8_32);
break;
+ case 560:
+ ret += tcrypt_test("ecdsa");
+ break;
+
+ case 561:
+#ifndef CONFIG_CRYPTO_FIPS
+ test_akcipher_speed("ecdsa", SIGN, sec,
+ ecdsa_speed_template, ECDSA_SPEED_VECTORS,
+ akc_speed_template_P192);
+ test_akcipher_speed("ecdsa", VERIFY, sec,
+ ecdsa_speed_template, ECDSA_SPEED_VECTORS,
+ akc_speed_template_P192);
+#endif
+ test_akcipher_speed("ecdsa", SIGN, sec,
+ ecdsa_speed_template, ECDSA_SPEED_VECTORS,
+ akc_speed_template_P256);
+ test_akcipher_speed("ecdsa", VERIFY, sec,
+ ecdsa_speed_template, ECDSA_SPEED_VECTORS,
+ akc_speed_template_P256);
+ break;
+
case 1000:
test_available();
break;
diff --git a/crypto/tcrypt.h b/crypto/tcrypt.h
index f0bfee1bb293..bd6a4b1cbcbe 100644
--- a/crypto/tcrypt.h
+++ b/crypto/tcrypt.h
@@ -7,6 +7,7 @@
* Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
* Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
* Copyright (c) 2007 Nokia Siemens Networks
+ * Copyright (c) 2017 NVIDIA Corporation
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
@@ -17,6 +18,16 @@
#ifndef _CRYPTO_TCRYPT_H
#define _CRYPTO_TCRYPT_H
+struct akcipher_speed_template {
+ unsigned char *key;
+ unsigned char *m;
+ unsigned char *c;
+ unsigned int key_len;
+ unsigned int m_size;
+ unsigned int c_size;
+ bool public_key_vec;
+};
+
struct cipher_speed_template {
const char *key;
unsigned int klen;
@@ -48,6 +59,117 @@ struct hash_speed {
};
/*
+ * ECDSA test vectors.
+ */
+#ifdef CONFIG_CRYPTO_FIPS
+#define ECDSA_SPEED_VECTORS 1
+#else
+#define ECDSA_SPEED_VECTORS 2
+#endif
+
+static struct akcipher_speed_template ecdsa_speed_template[] = {
+ {
+#ifndef CONFIG_CRYPTO_FIPS
+ /* [P-192,SHA-256] */
+ .m =
+ /* Msg / Hash */
+ "\xd0\xd8\xc0\x99\xe0\xe2\xf7\xf8"
+ "\x87\xe1\x6d\x11\xe1\xcc\x20\x43"
+ "\xaf\xc0\x80\xdb\x47\x72\xfa\xe3"
+ "\x95\xe5\xd1\x34\x7d\x31\xe8\x5a",
+ .m_size = 32,
+ .key =
+ /* version */
+ "\x01"
+ /* curve_id */
+ "\x01"
+ /* d */
+ "\x47\x7a\xf2\x5c\x86\xef\x09\x08"
+ "\xa4\x9a\x47\x53\x06\xfc\x61\xbc"
+ "\xa5\x6f\xdd\x7d\x2f\xd2\xed\x24"
+ /* Qx */
+ "\xdc\x14\xd4\xd8\x2e\x1e\x25\x2f"
+ "\x66\x28\xaa\x80\xbc\x38\x6a\x07"
+ "\x8a\x70\xb7\x74\x71\x2d\xf1\x9b"
+ /* Qy */
+ "\x98\x34\x57\x11\xb0\xdc\x3d\xff"
+ "\xfc\xdc\xfe\xa2\x1c\x47\x9e\x4e"
+ "\x82\x08\xfc\x7d\xd0\xc8\x54\x48",
+ .key_len = 74,
+ .c =
+ /* k */
+ "\x3e\x70\xc7\x86\xaf\xaa\x71\x7c"
+ "\x68\x96\xc5\xc3\xec\xb8\x29\xa3"
+ "\xfa\xf7\xa5\x36\xa2\x17\xc8\xa5"
+ /* R */
+ "\xf8\xef\x13\xa8\x86\xe6\x73\x85"
+ "\xdf\x2e\x88\x99\x91\x9b\xc2\x90"
+ "\xea\x1f\x36\xf4\xec\xba\x4a\x35"
+ /* S */
+ "\xc1\x82\x9e\x94\xb7\x58\x2c\x63"
+ "\x8e\xd7\x15\x5a\x38\x47\x30\x9b"
+ "\x1c\x11\x86\xac\x00\x00\xf5\x80",
+ .c_size = 72,
+ }, {
+#endif
+ /* [P-256,SHA-256] */
+ .m =
+ /* Msg / Hash */
+ "\x56\xec\x33\xa1\xa6\xe7\xc4\xdb"
+ "\x77\x03\x90\x1a\xfb\x2e\x1e\x4e"
+ "\x50\x09\xfe\x04\x72\x89\xc5\xc2"
+ "\x42\x13\x6c\xe3\xb7\xf6\xac\x44",
+ .m_size = 32,
+ .key =
+ /* version */
+ "\x01"
+ /* curve_id */
+ "\x02"
+ /* d */
+ "\x64\xb4\x72\xda\x6d\xa5\x54\xca"
+ "\xac\x3e\x4e\x0b\x13\xc8\x44\x5b"
+ "\x1a\x77\xf4\x59\xee\xa8\x4f\x1f"
+ "\x58\x8b\x5f\x71\x3d\x42\x9b\x51"
+ /* Qx */
+ "\x83\xbf\x71\xc2\x46\xff\x59\x3c"
+ "\x2f\xb1\xbf\x4b\xe9\x5d\x56\xd3"
+ "\xcc\x8f\xdb\x48\xa2\xbf\x33\xf0"
+ "\xf4\xc7\x5f\x07\x1c\xe9\xcb\x1c"
+ /* Qy */
+ "\xa9\x4c\x9a\xa8\x5c\xcd\x7c\xdc"
+ "\x78\x4e\x40\xb7\x93\xca\xb7\x6d"
+ "\xe0\x13\x61\x0e\x2c\xdb\x1f\x1a"
+ "\xa2\xf9\x11\x88\xc6\x14\x40\xce",
+ .key_len = 98,
+ .c =
+ /* k */
+ "\xde\x68\x2a\x64\x87\x07\x67\xb9"
+ "\x33\x5d\x4f\x82\x47\x62\x4a\x3b"
+ "\x7f\x3c\xe9\xf9\x45\xf2\x80\xa2"
+ "\x61\x6a\x90\x4b\xb1\xbb\xa1\x94"
+ /* R */
+ "\xac\xc2\xc8\x79\x6f\x5e\xbb\xca"
+ "\x7a\x5a\x55\x6a\x1f\x6b\xfd\x2a"
+ "\xed\x27\x95\x62\xd6\xe3\x43\x88"
+ "\x5b\x79\x14\xb5\x61\x80\xac\xf3"
+ /* S */
+ "\x03\x89\x05\xcc\x2a\xda\xcd\x3c"
+ "\x5a\x17\x6f\xe9\x18\xb2\x97\xef"
+ "\x1c\x37\xf7\x2b\x26\x76\x6c\x78"
+ "\xb2\xa6\x05\xca\x19\x78\xf7\x8b",
+ .c_size = 96,
+ },
+};
+
+/*
+ * AKCipher speed tests
+ */
+#ifndef CONFIG_CRYPTO_FIPS
+static u8 akc_speed_template_P192[] = {74, 0};
+#endif
+static u8 akc_speed_template_P256[] = {98, 0};
+
+/*
* Cipher speed tests
*/
static u8 speed_template_8[] = {8, 0};
--
1.7.6.3
^ permalink raw reply related
* [PATCH v2 5/6] crypto: testmgr: add ECDSA tests
From: Nitin Kumbhar @ 2017-02-03 11:12 UTC (permalink / raw)
To: herbert, davem; +Cc: linux-crypto, Nitin Kumbhar
In-Reply-To: <1486120375-13070-1-git-send-email-nkumbhar@nvidia.com>
Update crypto test manager to include NIST ECDSA
test vectors and various ECDSA tests. These include
tests for ECDSA signing, ECDSA sign-verification,
ECDSA signing and verifying generated signatures and
invalidation of incorrect signatures.
Signed-off-by: Nitin Kumbhar <nkumbhar@nvidia.com>
---
crypto/testmgr.c | 330 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
crypto/testmgr.h | 140 +++++++++++++++++++++++
2 files changed, 467 insertions(+), 3 deletions(-)
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 98eb09782db8..bed448e005b1 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -5,6 +5,7 @@
* Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
* Copyright (c) 2007 Nokia Siemens Networks
* Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au>
+ * Copyright (c) 2017 NVIDIA Corporation
*
* Updated RFC4106 AES-GCM testing.
* Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
@@ -2085,6 +2086,314 @@ static int alg_test_kpp(const struct alg_test_desc *desc, const char *driver,
return err;
}
+#define ECDSA_TEST_VALID_VERIFY 0
+#define ECDSA_TEST_INVALID_VERIFY 1
+#define ECDSA_TEST_SIGN_VERIFY 2
+#define ECDSA_TEST_SIGN 3
+
+static int __do_test_ecdsa_verify(struct crypto_akcipher *tfm,
+ struct akcipher_testvec *vec, int test)
+{
+ struct akcipher_request *req = NULL;
+ u8 *r_str = NULL, *s_str = NULL;
+ u8 *m_str = NULL;
+ struct scatterlist src_tab[3], dst;
+ struct tcrypt_result result;
+ unsigned int outbuf_maxlen;
+ u8 *outbuf = NULL;
+ unsigned int nbytes;
+ int err, m_size;
+
+ /* Alloc akcipher request */
+ req = akcipher_request_alloc(tfm, GFP_KERNEL);
+ if (!req)
+ return -ENOMEM;
+
+ /* Set private key */
+ err = crypto_akcipher_set_pub_key(tfm, vec->key, vec->key_len);
+ if (err)
+ goto error;
+
+ /*
+ * vec->c always contains k, R and S in that order. All are
+ * of same size and are equal to n i.e. the order of
+ * an elliptic curve.
+ */
+ nbytes = vec->c_size / 3;
+
+ r_str = kzalloc(nbytes, GFP_KERNEL);
+ s_str = kzalloc(nbytes, GFP_KERNEL);
+ m_str = kzalloc(vec->m_size, GFP_KERNEL);
+ if (!r_str || !s_str || !m_str) {
+ err = -ENOMEM;
+ goto error;
+ }
+ memcpy(r_str, (u8 *)vec->c + 1 * nbytes, nbytes);
+ memcpy(s_str, (u8 *)vec->c + 2 * nbytes, nbytes);
+ memcpy(m_str, vec->m, vec->m_size);
+
+ outbuf_maxlen = crypto_akcipher_maxsize(tfm);
+ if (outbuf_maxlen < 0) {
+ err = outbuf_maxlen;
+ goto error;
+ }
+ outbuf = kzalloc(outbuf_maxlen, GFP_KERNEL);
+ if (!outbuf) {
+ err = -ENOMEM;
+ goto error;
+ }
+
+ /* Intentionally set m_size to 8 to test invalid hash */
+ m_size = test == ECDSA_TEST_VALID_VERIFY ? vec->m_size : 8;
+
+ /* Set src and dst buffers */
+ sg_init_table(src_tab, 3);
+ sg_set_buf(&src_tab[0], m_str, m_size);
+ sg_set_buf(&src_tab[1], r_str, nbytes);
+ sg_set_buf(&src_tab[2], s_str, nbytes);
+ sg_init_one(&dst, outbuf, outbuf_maxlen);
+
+ akcipher_request_set_crypt(req, src_tab, &dst,
+ vec->m_size + 2 * nbytes, outbuf_maxlen);
+
+ /* Set up result callback */
+ init_completion(&result.completion);
+ akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+ tcrypt_complete, &result);
+
+ /* Run ecdsa verify operation on sig (r,s) */
+ err = wait_async_op(&result, crypto_akcipher_verify(req));
+error:
+ akcipher_request_free(req);
+ kfree(r_str);
+ kfree(s_str);
+ kfree(m_str);
+ kfree(outbuf);
+ return err;
+}
+
+static int do_test_ecdsa_verify(struct crypto_akcipher *tfm,
+ struct akcipher_testvec *vec)
+{
+ int err;
+
+ err = __do_test_ecdsa_verify(tfm, vec, ECDSA_TEST_VALID_VERIFY);
+ if (err)
+ pr_err("alg: ecdsa: verify(rs) test failed. err %d\n", err);
+
+ return err;
+}
+
+static int do_test_ecdsa_invalid_verify(struct crypto_akcipher *tfm,
+ struct akcipher_testvec *vec)
+{
+ int err;
+
+ err = __do_test_ecdsa_verify(tfm, vec, ECDSA_TEST_INVALID_VERIFY);
+ if (err != -EBADMSG) {
+ pr_err("alg: ecdsa: invalid verify test failed. err %d\n", err);
+ return err;
+ }
+
+ return 0;
+}
+
+static int __do_test_ecdsa_sign_verify(struct crypto_akcipher *tfm,
+ struct akcipher_testvec *vec,
+ int test)
+{
+ struct akcipher_request *req = NULL;
+ u8 *r_str = NULL, *s_str = NULL;
+ u8 *sig_r = NULL, *sig_s = NULL;
+ u8 *m_str = NULL, *k_str = NULL;
+ struct scatterlist src_tab[3];
+ struct scatterlist src, dst;
+ struct tcrypt_result result;
+ unsigned int outbuf_maxlen;
+ void *outbuf = NULL;
+ unsigned int nbytes;
+ int err;
+
+ /* Alloc akcipher request */
+ req = akcipher_request_alloc(tfm, GFP_KERNEL);
+ if (!req)
+ return -ENOMEM;
+
+ /* Set private key */
+ err = crypto_akcipher_set_priv_key(tfm, vec->key, vec->key_len);
+ if (err)
+ goto error;
+
+ /* Set pub key */
+ err = crypto_akcipher_set_pub_key(tfm, vec->key, vec->key_len);
+ if (err)
+ goto error;
+
+ /*
+ * vec->c always contains k, R and S in that order. All are
+ * of same size and are equal to n i.e. the order of
+ * an elliptic curve.
+ */
+ nbytes = vec->c_size / 3;
+
+ k_str = kzalloc(nbytes, GFP_KERNEL);
+ r_str = kzalloc(nbytes, GFP_KERNEL);
+ s_str = kzalloc(nbytes, GFP_KERNEL);
+ m_str = kzalloc(vec->m_size, GFP_KERNEL);
+ if (!k_str || !r_str || !s_str || !m_str) {
+ err = -ENOMEM;
+ goto error;
+ }
+ memcpy(k_str, (u8 *)vec->c + 0 * nbytes, nbytes);
+ memcpy(r_str, (u8 *)vec->c + 1 * nbytes, nbytes);
+ memcpy(s_str, (u8 *)vec->c + 2 * nbytes, nbytes);
+ memcpy(m_str, vec->m, vec->m_size);
+
+ outbuf_maxlen = crypto_akcipher_maxsize(tfm);
+ if (outbuf_maxlen < 0) {
+ err = outbuf_maxlen;
+ goto error;
+ }
+ outbuf = kzalloc(outbuf_maxlen, GFP_KERNEL);
+ if (!outbuf) {
+ err = -ENOMEM;
+ goto error;
+ }
+
+ sg_init_one(&src, m_str, vec->m_size);
+ sg_init_one(&dst, outbuf, outbuf_maxlen);
+
+ akcipher_request_set_crypt(req, &src, &dst,
+ vec->m_size, outbuf_maxlen);
+
+ /* Set up result callback */
+ init_completion(&result.completion);
+ akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+ tcrypt_complete, &result);
+
+ /* Set K in request for signing */
+ if (test == ECDSA_TEST_SIGN)
+ req->info = k_str;
+
+ /* Run ecdsa sign operation on message digest */
+ err = wait_async_op(&result, crypto_akcipher_sign(req));
+ if (err)
+ goto error;
+
+ /* verify that signature (r,s) is valid */
+ if (req->dst_len != 2 * nbytes) {
+ err = -EINVAL;
+ goto error;
+ }
+
+ /* outbuf contains r and s */
+ sig_r = outbuf;
+ sig_s = (u8 *)outbuf + nbytes;
+
+ if (test == ECDSA_TEST_SIGN) {
+ /* compare r & s */
+ if (memcmp(r_str, sig_r, nbytes) ||
+ memcmp(s_str, sig_s, nbytes)) {
+ err = -EINVAL;
+ goto error;
+ }
+ } else {
+ /* Set src and dst buffers */
+ sg_init_table(src_tab, 3);
+ sg_set_buf(&src_tab[0], m_str, vec->m_size);
+ sg_set_buf(&src_tab[1], sig_r, nbytes);
+ sg_set_buf(&src_tab[2], sig_s, nbytes);
+ sg_init_one(&dst, outbuf, outbuf_maxlen);
+
+ akcipher_request_set_crypt(req, src_tab, &dst,
+ vec->m_size + 2 * nbytes,
+ outbuf_maxlen);
+
+ /* Set up result callback */
+ init_completion(&result.completion);
+ akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+ tcrypt_complete, &result);
+
+ /* Run ecdsa verify operation on sig (r,s) */
+ err = wait_async_op(&result, crypto_akcipher_verify(req));
+ if (err)
+ goto error;
+ }
+
+error:
+ akcipher_request_free(req);
+ kfree(k_str);
+ kfree(r_str);
+ kfree(s_str);
+ kfree(m_str);
+ kfree(outbuf);
+ return err;
+}
+
+static int do_test_ecdsa_sign_verify(struct crypto_akcipher *tfm,
+ struct akcipher_testvec *vec)
+{
+ int err;
+
+ err = __do_test_ecdsa_sign_verify(tfm, vec, ECDSA_TEST_SIGN_VERIFY);
+ if (err)
+ pr_err("alg: ecdsa: sign/verify test failed. err %d\n", err);
+
+ return err;
+}
+
+static int do_test_ecdsa_sign(struct crypto_akcipher *tfm,
+ struct akcipher_testvec *vec)
+{
+ int err;
+
+ err = __do_test_ecdsa_sign_verify(tfm, vec, ECDSA_TEST_SIGN);
+ if (err)
+ pr_err("alg: ecdsa: sign test failed. err %d\n", err);
+
+ return err;
+}
+
+static int test_ecdsa_akcipher(struct crypto_akcipher *tfm, const char *alg,
+ struct akcipher_testvec *vecs, unsigned int tcount)
+{
+ int i, err = 0;
+
+ for (i = 0; i < tcount; i++) {
+ err = do_test_ecdsa_verify(tfm, &vecs[i]);
+ if (!err)
+ continue;
+
+ return err;
+ }
+
+ for (i = 0; i < tcount; i++) {
+ err = do_test_ecdsa_invalid_verify(tfm, &vecs[i]);
+ if (!err)
+ continue;
+
+ return err;
+ }
+
+ for (i = 0; i < tcount; i++) {
+ err = do_test_ecdsa_sign_verify(tfm, &vecs[i]);
+ if (!err)
+ continue;
+
+ return err;
+ }
+
+ for (i = 0; i < tcount; i++) {
+ err = do_test_ecdsa_sign(tfm, &vecs[i]);
+ if (!err)
+ continue;
+
+ return err;
+ }
+
+ return 0;
+}
+
static int test_akcipher_one(struct crypto_akcipher *tfm,
struct akcipher_testvec *vecs)
{
@@ -2236,9 +2545,17 @@ static int alg_test_akcipher(const struct alg_test_desc *desc,
driver, PTR_ERR(tfm));
return PTR_ERR(tfm);
}
- if (desc->suite.akcipher.vecs)
- err = test_akcipher(tfm, desc->alg, desc->suite.akcipher.vecs,
- desc->suite.akcipher.count);
+
+ if (desc->suite.akcipher.vecs) {
+ if (strncmp(desc->alg, "ecdsa", 5) == 0)
+ err = test_ecdsa_akcipher(tfm, desc->alg,
+ desc->suite.akcipher.vecs,
+ desc->suite.akcipher.count);
+ else
+ err = test_akcipher(tfm, desc->alg,
+ desc->suite.akcipher.vecs,
+ desc->suite.akcipher.count);
+ }
crypto_free_akcipher(tfm);
return err;
@@ -2982,6 +3299,13 @@ static int alg_test_null(const struct alg_test_desc *desc,
.kpp = __VECS(ecdh_tv_template)
}
}, {
+ .alg = "ecdsa",
+ .test = alg_test_akcipher,
+ .fips_allowed = 1,
+ .suite = {
+ .akcipher = __VECS(ecdsa_tv_template)
+ }
+ }, {
.alg = "gcm(aes)",
.test = alg_test_aead,
.fips_allowed = 1,
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 64595f067d72..00bb57f4707a 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -5,6 +5,7 @@
* Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
* Copyright (c) 2007 Nokia Siemens Networks
* Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au>
+ * Copyright (c) 2017 NVIDIA Corporation
*
* Updated RFC4106 AES-GCM testing. Some test vectors were taken from
* http://csrc.nist.gov/groups/ST/toolkit/BCM/documents/proposedmodes/
@@ -755,6 +756,145 @@ struct kpp_testvec dh_tv_template[] = {
}
};
+/*
+ * ECDSA NIST test vectors from SigGenComponent.txt file from
+ * 186-3ecdsasiggencomponenttestvectors.zip for P-192 and P-256
+ * elliptic curves.
+ */
+static struct akcipher_testvec ecdsa_tv_template[] = {
+ {
+#ifndef CONFIG_CRYPTO_FIPS
+ /* [P-192,SHA-1] */
+ .m =
+ /* Msg / Hash */
+ "\x92\x5b\xd6\xf4\x1c\x55\xbe\x3e"
+ "\x49\xb7\x16\xe6\x1d\x42\x12\x3f"
+ "\x42\x79\x80\x60",
+ .m_size = 20,
+ .key =
+ /* version */
+ "\x01"
+ /* curve_id */
+ "\x01"
+ /* d */
+ "\xf3\xd7\x60\xd6\x75\xf2\xcc\xeb"
+ "\xf0\xd2\xfd\xb3\xb9\x41\x3f\xb0"
+ "\xf8\x4f\x37\xd1\xb3\x37\x4f\xe1"
+ /* Qx */
+ "\xe6\x98\xcf\x5b\x2d\x2d\x98\x94"
+ "\x4c\x49\xa2\x80\x6e\x09\x32\x64"
+ "\xe7\xdb\x08\x0b\xa4\x8e\x00\x07"
+ /* Qy */
+ "\x77\x54\xd6\xe4\xf2\xd7\x1b\xc4"
+ "\x98\x6d\xe2\x5d\x21\xba\x36\xa6"
+ "\x4e\x41\x0b\xd0\x81\xb6\xfa\x76",
+ .key_len = 74,
+ .c =
+ /* k */
+ "\x25\x5f\x68\x89\xa2\x31\xbc\x57"
+ "\x4d\x15\xc4\x12\xfb\x56\x45\x68"
+ "\x83\x07\xa1\x43\x70\xbc\x0a\xcb"
+ /* R */
+ "\x3e\xa6\x58\x62\xb4\x98\x96\x1a"
+ "\xf9\xf2\x5b\xec\x55\xf8\xdd\xff"
+ "\x93\xd7\xd0\xbd\x62\xd9\x94\x69"
+ /* S */
+ "\x41\x9f\x1a\x0e\xc0\x5f\xcf\x73"
+ "\x5b\x40\x21\x85\xbc\x02\xab\x44"
+ "\x37\x90\x34\xa2\x65\x64\xba\x02",
+ .c_size = 72,
+ }, {
+ /* [P-192,SHA-256] */
+ .m =
+ /* Msg / Hash */
+ "\xd0\xd8\xc0\x99\xe0\xe2\xf7\xf8"
+ "\x87\xe1\x6d\x11\xe1\xcc\x20\x43"
+ "\xaf\xc0\x80\xdb\x47\x72\xfa\xe3"
+ "\x95\xe5\xd1\x34\x7d\x31\xe8\x5a",
+ .m_size = 32,
+ .key =
+ /* version */
+ "\x01"
+ /* curve_id */
+ "\x01"
+ /* d */
+ "\x47\x7a\xf2\x5c\x86\xef\x09\x08"
+ "\xa4\x9a\x47\x53\x06\xfc\x61\xbc"
+ "\xa5\x6f\xdd\x7d\x2f\xd2\xed\x24"
+ /* Qx */
+ "\xdc\x14\xd4\xd8\x2e\x1e\x25\x2f"
+ "\x66\x28\xaa\x80\xbc\x38\x6a\x07"
+ "\x8a\x70\xb7\x74\x71\x2d\xf1\x9b"
+ /* Qy */
+ "\x98\x34\x57\x11\xb0\xdc\x3d\xff"
+ "\xfc\xdc\xfe\xa2\x1c\x47\x9e\x4e"
+ "\x82\x08\xfc\x7d\xd0\xc8\x54\x48",
+ .key_len = 74,
+ .c =
+ /* k */
+ "\x3e\x70\xc7\x86\xaf\xaa\x71\x7c"
+ "\x68\x96\xc5\xc3\xec\xb8\x29\xa3"
+ "\xfa\xf7\xa5\x36\xa2\x17\xc8\xa5"
+ /* R */
+ "\xf8\xef\x13\xa8\x86\xe6\x73\x85"
+ "\xdf\x2e\x88\x99\x91\x9b\xc2\x90"
+ "\xea\x1f\x36\xf4\xec\xba\x4a\x35"
+ /* S */
+ "\xc1\x82\x9e\x94\xb7\x58\x2c\x63"
+ "\x8e\xd7\x15\x5a\x38\x47\x30\x9b"
+ "\x1c\x11\x86\xac\x00\x00\xf5\x80",
+ .c_size = 72,
+ }, {
+#endif
+ /* [P-256,SHA-256] */
+ .m =
+ /* Msg / Hash */
+ "\x56\xec\x33\xa1\xa6\xe7\xc4\xdb"
+ "\x77\x03\x90\x1a\xfb\x2e\x1e\x4e"
+ "\x50\x09\xfe\x04\x72\x89\xc5\xc2"
+ "\x42\x13\x6c\xe3\xb7\xf6\xac\x44",
+ .m_size = 32,
+ .key =
+ /* version */
+ "\x01"
+ /* curve_id */
+ "\x02"
+ /* d */
+ "\x64\xb4\x72\xda\x6d\xa5\x54\xca"
+ "\xac\x3e\x4e\x0b\x13\xc8\x44\x5b"
+ "\x1a\x77\xf4\x59\xee\xa8\x4f\x1f"
+ "\x58\x8b\x5f\x71\x3d\x42\x9b\x51"
+ /* Qx */
+ "\x83\xbf\x71\xc2\x46\xff\x59\x3c"
+ "\x2f\xb1\xbf\x4b\xe9\x5d\x56\xd3"
+ "\xcc\x8f\xdb\x48\xa2\xbf\x33\xf0"
+ "\xf4\xc7\x5f\x07\x1c\xe9\xcb\x1c"
+ /* Qy */
+ "\xa9\x4c\x9a\xa8\x5c\xcd\x7c\xdc"
+ "\x78\x4e\x40\xb7\x93\xca\xb7\x6d"
+ "\xe0\x13\x61\x0e\x2c\xdb\x1f\x1a"
+ "\xa2\xf9\x11\x88\xc6\x14\x40\xce",
+ .key_len = 98,
+ .c =
+ /* k */
+ "\xde\x68\x2a\x64\x87\x07\x67\xb9"
+ "\x33\x5d\x4f\x82\x47\x62\x4a\x3b"
+ "\x7f\x3c\xe9\xf9\x45\xf2\x80\xa2"
+ "\x61\x6a\x90\x4b\xb1\xbb\xa1\x94"
+ /* R */
+ "\xac\xc2\xc8\x79\x6f\x5e\xbb\xca"
+ "\x7a\x5a\x55\x6a\x1f\x6b\xfd\x2a"
+ "\xed\x27\x95\x62\xd6\xe3\x43\x88"
+ "\x5b\x79\x14\xb5\x61\x80\xac\xf3"
+ /* S */
+ "\x03\x89\x05\xcc\x2a\xda\xcd\x3c"
+ "\x5a\x17\x6f\xe9\x18\xb2\x97\xef"
+ "\x1c\x37\xf7\x2b\x26\x76\x6c\x78"
+ "\xb2\xa6\x05\xca\x19\x78\xf7\x8b",
+ .c_size = 96,
+ },
+};
+
struct kpp_testvec ecdh_tv_template[] = {
{
#ifndef CONFIG_CRYPTO_FIPS
--
1.7.6.3
^ permalink raw reply related
* [PATCH v2 4/6] crypto: ecdsa: add ECDSA SW implementation
From: Nitin Kumbhar @ 2017-02-03 11:12 UTC (permalink / raw)
To: herbert, davem; +Cc: linux-crypto, Nitin Kumbhar
In-Reply-To: <1486120375-13070-1-git-send-email-nkumbhar@nvidia.com>
This adds support for ECDSA algorithm. This implementation supports
sign and verify functions for ECDSA algorithm using akcipher. As ECDSA
is a signing algorithm dummy functions are added for encrypt() and
decrypt().
Helper routines for parsing public and private ECC keys for ECDSA are
added as well.
Signed-off-by: Nitin Kumbhar <nkumbhar@nvidia.com>
---
crypto/Kconfig | 7 +
crypto/Makefile | 3 +
crypto/ecdsa.c | 362 +++++++++++++++++++++++++++++++++++++++++++++
crypto/ecdsa_helper.c | 116 +++++++++++++++
include/crypto/akcipher.h | 5 +-
include/crypto/ecdsa.h | 81 ++++++++++
6 files changed, 573 insertions(+), 1 deletions(-)
create mode 100644 crypto/ecdsa.c
create mode 100644 crypto/ecdsa_helper.c
create mode 100644 include/crypto/ecdsa.h
diff --git a/crypto/Kconfig b/crypto/Kconfig
index e240075d6f46..1c5c236b3bbc 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -140,6 +140,13 @@ config CRYPTO_ECDH
help
Generic implementation of the ECDH algorithm
+config CRYPTO_ECDSA
+ tristate "ECDSA algorithm"
+ select CRYPTO_AKCIPHER
+ select CRYPTO_ECC
+ help
+ Generic implementation of the ECDSA algorithm
+
config CRYPTO_MANAGER
tristate "Cryptographic algorithm manager"
select CRYPTO_MANAGER2
diff --git a/crypto/Makefile b/crypto/Makefile
index 827740a47a37..9c13eb2ade6a 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -38,6 +38,9 @@ obj-$(CONFIG_CRYPTO_ECC) += ecc.o
ecdh_generic-y := ecdh.o
ecdh_generic-y += ecdh_helper.o
obj-$(CONFIG_CRYPTO_ECDH) += ecdh_generic.o
+ecdsa_generic-y := ecdsa.o
+ecdsa_generic-y += ecdsa_helper.o
+obj-$(CONFIG_CRYPTO_ECDSA) += ecdsa_generic.o
$(obj)/rsapubkey-asn1.o: $(obj)/rsapubkey-asn1.c $(obj)/rsapubkey-asn1.h
$(obj)/rsaprivkey-asn1.o: $(obj)/rsaprivkey-asn1.c $(obj)/rsaprivkey-asn1.h
diff --git a/crypto/ecdsa.c b/crypto/ecdsa.c
new file mode 100644
index 000000000000..178528ec65d2
--- /dev/null
+++ b/crypto/ecdsa.c
@@ -0,0 +1,362 @@
+/*
+ * ECDSA generic algorithm
+ *
+ * Copyright (c) 2017, NVIDIA Corporation. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+#include <crypto/rng.h>
+#include <crypto/internal/akcipher.h>
+#include <crypto/akcipher.h>
+#include <crypto/ecdsa.h>
+
+#include "ecc.h"
+
+struct ecdsa_ctx {
+ unsigned int curve_id;
+ unsigned int ndigits;
+ u64 private_key[ECC_MAX_DIGITS];
+ u64 public_key[2 * ECC_MAX_DIGITS];
+};
+
+static inline struct ecdsa_ctx *ecdsa_get_ctx(struct crypto_akcipher *tfm)
+{
+ return akcipher_tfm_ctx(tfm);
+}
+
+static void ecdsa_parse_msg_hash(struct akcipher_request *req, u64 *msg,
+ unsigned int ndigits)
+{
+ unsigned int nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+ unsigned int hash_len, hash_off;
+ unsigned char *hash, *msg_ptr;
+ int i;
+
+ /*
+ * If hash_len == nbytes:
+ * copy nbytes from req
+ * If hash_len > nbytes:
+ * copy left most nbytes from hash ignoring LSBs
+ * If hash_len < nbytes:
+ * copy hash_len from req and zero remaining bytes
+ * (nbytes - hash_len)
+ */
+ hash_len = req->src[0].length;
+ hash_off = hash_len <= nbytes ? 0 : hash_len - nbytes;
+
+ msg_ptr = (unsigned char *)msg;
+ hash = sg_virt(&req->src[0]);
+
+ for (i = hash_off; i < hash_len; i++)
+ *msg_ptr++ = hash[i];
+ for (; i < nbytes; i++)
+ *msg_ptr++ = 0;
+}
+
+static int ecdsa_get_rnd_bytes(u8 *rdata, unsigned int dlen)
+{
+ int err;
+
+ err = crypto_get_default_rng();
+ if (err)
+ return err;
+
+ err = crypto_rng_get_bytes(crypto_default_rng, rdata, dlen);
+ crypto_put_default_rng();
+ return err;
+}
+
+int ecdsa_sign(struct akcipher_request *req)
+{
+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+ struct ecdsa_ctx *ctx = ecdsa_get_ctx(tfm);
+ unsigned int ndigits = ctx->ndigits;
+ unsigned int nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+ unsigned int curve_id = ctx->curve_id;
+ const struct ecc_curve *curve = ecc_get_curve(curve_id);
+ struct ecc_point *x1y1 = NULL;
+ u64 z[ndigits], d[ndigits];
+ u64 k[ndigits], k_inv[ndigits];
+ u64 r[ndigits], s[ndigits];
+ u64 dr[ndigits], zdr[ndigits];
+ u8 *r_ptr, *s_ptr;
+ int err;
+
+ if (req->dst_len < 2 * nbytes) {
+ req->dst_len = 2 * nbytes;
+ return -EINVAL;
+ }
+
+ if (!curve)
+ return -EINVAL;
+
+ ecdsa_parse_msg_hash(req, z, ndigits);
+
+ /* d */
+ vli_set(d, (const u64 *)ctx->private_key, ndigits);
+
+ /* k */
+ err = ecdsa_get_rnd_bytes((u8 *)k, nbytes);
+ if (err)
+ return err;
+
+#if defined(CONFIG_CRYPTO_MANAGER2)
+ if (req->info)
+ vli_copy_from_buf(k, ndigits, req->info, nbytes);
+#endif
+
+ x1y1 = ecc_alloc_point(ndigits);
+ if (!x1y1)
+ return -ENOMEM;
+
+ /* (x1, y1) = k x G */
+ ecc_point_mult(x1y1, &curve->g, k, NULL, curve->p, ndigits);
+
+ /* r = x1 mod n */
+ vli_mod(r, x1y1->x, curve->n, ndigits);
+
+ /* k^-1 */
+ vli_mod_inv(k_inv, k, curve->n, ndigits);
+
+ /* d . r mod n */
+ vli_mod_mult(dr, d, r, curve->n, ndigits);
+
+ /* z + dr mod n */
+ vli_mod_add(zdr, z, dr, curve->n, ndigits);
+
+ /* k^-1 . ( z + dr) mod n */
+ vli_mod_mult(s, k_inv, zdr, curve->n, ndigits);
+
+ /* write signature (r,s) in dst */
+ r_ptr = sg_virt(req->dst);
+ s_ptr = (u8 *)sg_virt(req->dst) + nbytes;
+
+ vli_copy_to_buf(r_ptr, nbytes, r, ndigits);
+ vli_copy_to_buf(s_ptr, nbytes, s, ndigits);
+
+ req->dst_len = 2 * nbytes;
+
+ ecc_free_point(x1y1);
+ return 0;
+}
+
+int ecdsa_verify(struct akcipher_request *req)
+{
+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+ struct ecdsa_ctx *ctx = ecdsa_get_ctx(tfm);
+ unsigned int ndigits = ctx->ndigits;
+ unsigned int nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+ unsigned int curve_id = ctx->curve_id;
+ const struct ecc_curve *curve = ecc_get_curve(curve_id);
+ struct ecc_point *x1y1 = NULL, *x2y2 = NULL, *Q = NULL;
+ u64 r[ndigits], s[ndigits], v[ndigits];
+ u64 z[ndigits], w[ndigits];
+ u64 u1[ndigits], u2[ndigits];
+ u64 x1[ndigits], x2[ndigits];
+ u64 y1[ndigits], y2[ndigits];
+ u64 *ctx_qx, *ctx_qy;
+ int ret;
+
+ if (!curve)
+ return -EINVAL;
+
+ x1y1 = ecc_alloc_point(ndigits);
+ x2y2 = ecc_alloc_point(ndigits);
+ Q = ecc_alloc_point(ndigits);
+ if (!x1y1 || !x2y2 || !Q) {
+ ret = -ENOMEM;
+ goto exit;
+ }
+
+ ecdsa_parse_msg_hash(req, z, ndigits);
+
+ /* Signature r,s */
+ vli_copy_from_buf(r, ndigits, sg_virt(&req->src[1]), nbytes);
+ vli_copy_from_buf(s, ndigits, sg_virt(&req->src[2]), nbytes);
+
+ /* w = s^-1 mod n */
+ vli_mod_inv(w, s, curve->n, ndigits);
+
+ /* u1 = zw mod n */
+ vli_mod_mult(u1, z, w, curve->n, ndigits);
+
+ /* u2 = rw mod n */
+ vli_mod_mult(u2, r, w, curve->n, ndigits);
+
+ /* u1 . G */
+ ecc_point_mult(x1y1, &curve->g, u1, NULL, curve->p, ndigits);
+
+ /* Q=(Qx,Qy) */
+ ctx_qx = ctx->public_key;
+ ctx_qy = ctx_qx + ECC_MAX_DIGITS;
+ vli_set(Q->x, ctx_qx, ndigits);
+ vli_set(Q->y, ctx_qy, ndigits);
+
+ /* u2 x Q */
+ ecc_point_mult(x2y2, Q, u2, NULL, curve->p, ndigits);
+
+ vli_set(x1, x1y1->x, ndigits);
+ vli_set(y1, x1y1->y, ndigits);
+ vli_set(x2, x2y2->x, ndigits);
+ vli_set(y2, x2y2->y, ndigits);
+
+ /* x1y1 + x2y2 => P + Q; P + Q in x2 y2 */
+ ecc_point_add(x1, y1, x2, y2, curve->p, ndigits);
+
+ /* v = x mod n */
+ vli_mod(v, x2, curve->n, ndigits);
+
+ /* validate signature */
+ ret = vli_cmp(v, r, ndigits) == 0 ? 0 : -EBADMSG;
+ exit:
+ ecc_free_point(x1y1);
+ ecc_free_point(x2y2);
+ ecc_free_point(Q);
+ return ret;
+}
+
+int ecdsa_dummy_enc(struct akcipher_request *req)
+{
+ return -EINVAL;
+}
+
+int ecdsa_dummy_dec(struct akcipher_request *req)
+{
+ return -EINVAL;
+}
+
+int ecdsa_set_pub_key(struct crypto_akcipher *tfm, const void *key,
+ unsigned int keylen)
+{
+ struct ecdsa_ctx *ctx = ecdsa_get_ctx(tfm);
+ struct ecdsa params;
+ unsigned int ndigits;
+ unsigned int nbytes;
+ u8 *params_qx, *params_qy;
+ u64 *ctx_qx, *ctx_qy;
+ int err = 0;
+
+ if (crypto_ecdsa_parse_pub_key(key, keylen, ¶ms))
+ return -EINVAL;
+
+ ndigits = ecdsa_supported_curve(params.curve_id);
+ if (!ndigits)
+ return -EINVAL;
+
+ err = ecc_is_pub_key_valid(params.curve_id, ndigits,
+ params.key, params.key_size);
+ if (err)
+ return err;
+
+ ctx->curve_id = params.curve_id;
+ ctx->ndigits = ndigits;
+ nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+
+ params_qx = params.key;
+ params_qy = params_qx + ECC_MAX_DIGIT_BYTES;
+
+ ctx_qx = ctx->public_key;
+ ctx_qy = ctx_qx + ECC_MAX_DIGITS;
+
+ vli_copy_from_buf(ctx_qx, ndigits, params_qx, nbytes);
+ vli_copy_from_buf(ctx_qy, ndigits, params_qy, nbytes);
+
+ memset(¶ms, 0, sizeof(params));
+ return 0;
+}
+
+int ecdsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,
+ unsigned int keylen)
+{
+ struct ecdsa_ctx *ctx = ecdsa_get_ctx(tfm);
+ struct ecdsa params;
+ unsigned int ndigits;
+ unsigned int nbytes;
+
+ if (crypto_ecdsa_parse_priv_key(key, keylen, ¶ms))
+ return -EINVAL;
+
+ ndigits = ecdsa_supported_curve(params.curve_id);
+ if (!ndigits)
+ return -EINVAL;
+
+ ctx->curve_id = params.curve_id;
+ ctx->ndigits = ndigits;
+ nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+
+ if (ecc_is_key_valid(ctx->curve_id, ctx->ndigits,
+ (const u8 *)params.key, params.key_size) < 0)
+ return -EINVAL;
+
+ vli_copy_from_buf(ctx->private_key, ndigits, params.key, nbytes);
+
+ memset(¶ms, 0, sizeof(params));
+ return 0;
+}
+
+int ecdsa_max_size(struct crypto_akcipher *tfm)
+{
+ struct ecdsa_ctx *ctx = ecdsa_get_ctx(tfm);
+ int nbytes = ctx->ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+
+ /* For r,s */
+ return 2 * nbytes;
+}
+
+int ecdsa_init_tfm(struct crypto_akcipher *tfm)
+{
+ return 0;
+}
+
+void ecdsa_exit_tfm(struct crypto_akcipher *tfm)
+{
+}
+
+static struct akcipher_alg ecdsa_alg = {
+ .sign = ecdsa_sign,
+ .verify = ecdsa_verify,
+ .encrypt = ecdsa_dummy_enc,
+ .decrypt = ecdsa_dummy_dec,
+ .set_priv_key = ecdsa_set_priv_key,
+ .set_pub_key = ecdsa_set_pub_key,
+ .max_size = ecdsa_max_size,
+ .init = ecdsa_init_tfm,
+ .exit = ecdsa_exit_tfm,
+ .base = {
+ .cra_name = "ecdsa",
+ .cra_driver_name = "ecdsa-generic",
+ .cra_priority = 100,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct ecdsa_ctx),
+ },
+};
+
+static int ecdsa_init(void)
+{
+ int ret;
+
+ ret = crypto_register_akcipher(&ecdsa_alg);
+ if (ret)
+ pr_err("ecdsa alg register failed. err:%d\n", ret);
+ return ret;
+}
+
+static void ecdsa_exit(void)
+{
+ crypto_unregister_akcipher(&ecdsa_alg);
+}
+
+module_init(ecdsa_init);
+module_exit(ecdsa_exit);
+
+MODULE_ALIAS_CRYPTO("ecdsa");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("ECDSA Generic Algorithm");
+MODULE_AUTHOR("NVIDIA Corporation");
diff --git a/crypto/ecdsa_helper.c b/crypto/ecdsa_helper.c
new file mode 100644
index 000000000000..d31eb54431a9
--- /dev/null
+++ b/crypto/ecdsa_helper.c
@@ -0,0 +1,116 @@
+/*
+ * ECDSA helper routines
+ *
+ * Copyright (c) 2017, NVIDIA Corporation. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/export.h>
+#include <linux/err.h>
+#include <linux/string.h>
+#include <crypto/ecdsa.h>
+
+#include "ecc.h"
+
+#define ECDSA_KEY_MIN_SIZE (1 + 1 + 24) /* ver + cid + n (P-192) */
+
+unsigned int ecdsa_supported_curve(unsigned int curve_id)
+{
+ switch (curve_id) {
+ case ECC_CURVE_NIST_P192: return 3;
+ case ECC_CURVE_NIST_P256: return 4;
+ default: return 0;
+ }
+}
+
+static inline u8 *ecdsa_pack_data(void *dst, const void *src, size_t sz)
+{
+ memcpy(dst, src, sz);
+ return dst + sz;
+}
+
+static inline const u8 *ecdsa_unpack_data(void *dst, const void *src, size_t sz)
+{
+ memcpy(dst, src, sz);
+ return src + sz;
+}
+
+int crypto_ecdsa_parse_pub_key(const char *buf, unsigned int len,
+ struct ecdsa *params)
+{
+ unsigned char version;
+ unsigned int ndigits;
+ unsigned int nbytes;
+ const u8 *ptr = buf;
+ u8 *qx, *qy;
+
+ if (unlikely(!buf) || len < ECDSA_KEY_MIN_SIZE)
+ return -EINVAL;
+
+ ptr = ecdsa_unpack_data(&version, ptr, sizeof(version));
+ if (version != 1)
+ return -EINVAL;
+
+ ptr = ecdsa_unpack_data(¶ms->curve_id, ptr,
+ sizeof(params->curve_id));
+
+ ndigits = ecdsa_supported_curve(params->curve_id);
+ if (!ndigits)
+ return -EINVAL;
+
+ nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+
+ /* skip private key */
+ ptr = ecdsa_unpack_data(¶ms->key, ptr, nbytes);
+
+ /* copy public key */
+ qx = params->key;
+ qy = qx + ECC_MAX_DIGIT_BYTES;
+
+ ptr = ecdsa_unpack_data(qx, ptr, nbytes);
+ ptr = ecdsa_unpack_data(qy, ptr, nbytes);
+
+ params->key_size = 2 * nbytes;
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(crypto_ecdsa_parse_pub_key);
+
+int crypto_ecdsa_parse_priv_key(const char *buf, unsigned int len,
+ struct ecdsa *params)
+{
+ unsigned char version;
+ unsigned int ndigits;
+ unsigned int nbytes;
+ const u8 *ptr = buf;
+
+ if (unlikely(!buf) || len < ECDSA_KEY_MIN_SIZE)
+ return -EINVAL;
+
+ ptr = ecdsa_unpack_data(&version, ptr, sizeof(version));
+ if (version != 1)
+ return -EINVAL;
+
+ ptr = ecdsa_unpack_data(¶ms->curve_id, ptr,
+ sizeof(params->curve_id));
+
+ ndigits = ecdsa_supported_curve(params->curve_id);
+ if (!ndigits)
+ return -EINVAL;
+
+ nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+
+ params->key_size = nbytes;
+
+ /* copy private key */
+ ptr = ecdsa_unpack_data(¶ms->key, ptr, nbytes);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(crypto_ecdsa_parse_priv_key);
diff --git a/include/crypto/akcipher.h b/include/crypto/akcipher.h
index c37cc59e9bf2..6b34e9043a6f 100644
--- a/include/crypto/akcipher.h
+++ b/include/crypto/akcipher.h
@@ -3,6 +3,7 @@
*
* Copyright (c) 2015, Intel Corporation
* Authors: Tadeusz Struk <tadeusz.struk@intel.com>
+ * Copyright (c) 2017, NVIDIA Corporation. All Rights Reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
@@ -27,6 +28,7 @@
* result.
* In case of error where the dst sgl size was insufficient,
* it will be updated to the size required for the operation.
+ * @info: Any request specific data needed to process the request.
* @__ctx: Start of private context data
*/
struct akcipher_request {
@@ -35,6 +37,7 @@ struct akcipher_request {
struct scatterlist *dst;
unsigned int src_len;
unsigned int dst_len;
+ void *info;
void *__ctx[] CRYPTO_MINALIGN_ATTR;
};
@@ -193,7 +196,7 @@ static inline void crypto_free_akcipher(struct crypto_akcipher *tfm)
{
struct akcipher_request *req;
- req = kmalloc(sizeof(*req) + crypto_akcipher_reqsize(tfm), gfp);
+ req = kzalloc(sizeof(*req) + crypto_akcipher_reqsize(tfm), gfp);
if (likely(req))
akcipher_request_set_tfm(req, tfm);
diff --git a/include/crypto/ecdsa.h b/include/crypto/ecdsa.h
new file mode 100644
index 000000000000..6923d0268770
--- /dev/null
+++ b/include/crypto/ecdsa.h
@@ -0,0 +1,81 @@
+/*
+ * ECC parameters for ECDSA
+ *
+ * Copyright (c) 2017, NVIDIA Corporation. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+
+#ifndef _CRYPTO_ECDSA_
+#define _CRYPTO_ECDSA_
+
+#include <crypto/ecc.h>
+
+/**
+ * DOC: ECDSA Helper Functions
+ *
+ * To use ECDSA as a akcipher, following functions should be used
+ * along with ECDSA private/public keys. The keys are mentioned as
+ * a packet private-public key and can be set with API functions
+ * crypto_akcipher_set_priv_key() & crypto_akcipher_set_pub_key().
+ */
+
+/**
+ * struct ecdsa - define an ECDSA private or public key
+ *
+ * @curve_id: ECC curve id the keys are based on
+ * @key: Private or public ECDSA key. Private key shall be a valid
+ * number as per curve's prime. Public key is expressed by
+ * valid affine coordinates Qx & Qy.
+ * @key_size: Size of ECDSA private/public key
+ */
+struct ecdsa {
+ unsigned char curve_id;
+ unsigned char key[2 * ECC_MAX_DIGIT_BYTES];
+ unsigned short key_size;
+};
+
+/**
+ * crypto_ecdsa_parse_pub_key() - parse and obtain ECDSA public key
+ * @buf: Buffer holding ECDDA packet key that should be parsed
+ * to get ECDSA public key
+ * @len: Length of the packet private-public key buffer
+ * @params: Buffer allocated by the caller that is filled with
+ * ECDSA public key
+ *
+ * This routine parses packet key from @buf and obtains version, curve id,
+ * private key and public key. It checks for correct version and supported
+ * curve id. It copies public key from the public key location in given
+ * ECDSA packet key to @params.
+ *
+ * Return: -EINVAL on errors, 0 on success
+ */
+int crypto_ecdsa_parse_pub_key(const char *buf, unsigned int len,
+ struct ecdsa *params);
+
+/**
+ * crypto_ecdsa_parse_priv_key() - parse and obtain ECDSA private key
+ * @buf: Buffer holding ECDDA packet key that should be parsed
+ * to get ECDSA private key
+ * @len: Length of the packet private-public key buffer
+ * @params: Buffer allocated by the caller that is filled with
+ * ECDSA private key
+ *
+ * Return: -EINVAL on errors, 0 on success
+ */
+int crypto_ecdsa_parse_priv_key(const char *buf, unsigned int len,
+ struct ecdsa *params);
+
+/**
+ * ecdsa_supported_curve() - check supported curve
+ * @curve_id: ECC curve id as defined by kernel
+ *
+ * Return: 0 for un-supported curve, ECC DIGITS for curve on success
+ */
+unsigned int ecdsa_supported_curve(unsigned int curve_id);
+
+#endif /* _CRYPTO_ECDSA_ */
--
1.7.6.3
^ permalink raw reply related
* [PATCH v2 3/6] crypto: ecc: export vli and ecc ops
From: Nitin Kumbhar @ 2017-02-03 11:12 UTC (permalink / raw)
To: herbert, davem; +Cc: linux-crypto, Nitin Kumbhar
In-Reply-To: <1486120375-13070-1-git-send-email-nkumbhar@nvidia.com>
Export vli and ECC related functions so that these can
be used by all ECC algorithm modules like ECDH, ECDSA and ECIES.
Signed-off-by: Nitin Kumbhar <nkumbhar@nvidia.com>
---
crypto/ecc.c | 115 +++++++++++++++++++++++++++++++++++++---------------------
crypto/ecc.h | 56 ++++++++++++++++++++++++++++
2 files changed, 130 insertions(+), 41 deletions(-)
diff --git a/crypto/ecc.c b/crypto/ecc.c
index 1b8e8d248859..04aaa5b37aa6 100644
--- a/crypto/ecc.c
+++ b/crypto/ecc.c
@@ -77,7 +77,7 @@
.n = nist_p256_n
};
-static inline const struct ecc_curve *ecc_get_curve(unsigned int curve_id)
+const struct ecc_curve *ecc_get_curve(unsigned int curve_id)
{
switch (curve_id) {
/* In FIPS mode only allow P256 and higher */
@@ -89,6 +89,7 @@
return NULL;
}
}
+EXPORT_SYMBOL_GPL(ecc_get_curve);
static u64 *ecc_alloc_digits_space(unsigned int ndigits)
{
@@ -105,7 +106,7 @@ static void ecc_free_digits_space(u64 *space)
kzfree(space);
}
-static struct ecc_point *ecc_alloc_point(unsigned int ndigits)
+struct ecc_point *ecc_alloc_point(unsigned int ndigits)
{
struct ecc_point *p = kmalloc(sizeof(*p), GFP_KERNEL);
@@ -130,8 +131,9 @@ static void ecc_free_digits_space(u64 *space)
kfree(p);
return NULL;
}
+EXPORT_SYMBOL_GPL(ecc_alloc_point);
-static void ecc_free_point(struct ecc_point *p)
+void ecc_free_point(struct ecc_point *p)
{
if (!p)
return;
@@ -140,17 +142,19 @@ static void ecc_free_point(struct ecc_point *p)
kzfree(p->y);
kzfree(p);
}
+EXPORT_SYMBOL_GPL(ecc_free_point);
-static void vli_clear(u64 *vli, unsigned int ndigits)
+void vli_clear(u64 *vli, unsigned int ndigits)
{
int i;
for (i = 0; i < ndigits; i++)
vli[i] = 0;
}
+EXPORT_SYMBOL_GPL(vli_clear);
/* Returns true if vli == 0, false otherwise. */
-static bool vli_is_zero(const u64 *vli, unsigned int ndigits)
+bool vli_is_zero(const u64 *vli, unsigned int ndigits)
{
int i;
@@ -161,15 +165,17 @@ static bool vli_is_zero(const u64 *vli, unsigned int ndigits)
return true;
}
+EXPORT_SYMBOL_GPL(vli_is_zero);
/* Returns nonzero if bit bit of vli is set. */
-static u64 vli_test_bit(const u64 *vli, unsigned int bit)
+u64 vli_test_bit(const u64 *vli, unsigned int bit)
{
return (vli[bit / 64] & ((u64)1 << (bit % 64)));
}
+EXPORT_SYMBOL_GPL(vli_test_bit);
/* Counts the number of 64-bit "digits" in vli. */
-static unsigned int vli_num_digits(const u64 *vli, unsigned int ndigits)
+unsigned int vli_num_digits(const u64 *vli, unsigned int ndigits)
{
int i;
@@ -181,9 +187,10 @@ static unsigned int vli_num_digits(const u64 *vli, unsigned int ndigits)
return (i + 1);
}
+EXPORT_SYMBOL_GPL(vli_num_digits);
/* Counts the number of bits required for vli. */
-static unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits)
+unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits)
{
unsigned int i, num_digits;
u64 digit;
@@ -198,15 +205,17 @@ static unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits)
return ((num_digits - 1) * 64 + i);
}
+EXPORT_SYMBOL_GPL(vli_num_bits);
/* Sets dest = src. */
-static void vli_set(u64 *dest, const u64 *src, unsigned int ndigits)
+void vli_set(u64 *dest, const u64 *src, unsigned int ndigits)
{
int i;
for (i = 0; i < ndigits; i++)
dest[i] = src[i];
}
+EXPORT_SYMBOL_GPL(vli_set);
/* Copy from vli to buf.
* For buffers smaller than vli: copy only LSB nbytes from vli.
@@ -225,6 +234,7 @@ void vli_copy_to_buf(u8 *dst_buf, unsigned int buf_len,
for (; i < buf_len; i++)
dst_buf[i] = 0;
}
+EXPORT_SYMBOL_GPL(vli_copy_to_buf);
/* Copy from buffer to vli.
* For buffers smaller than vli: fill up remaining vli with zeroes.
@@ -243,9 +253,10 @@ void vli_copy_from_buf(u64 *dst_vli, unsigned int ndigits,
for (; i < nbytes; i++)
vli[i] = 0;
}
+EXPORT_SYMBOL_GPL(vli_copy_from_buf);
/* Returns sign of left - right. */
-static int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits)
+int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits)
{
int i;
@@ -258,12 +269,13 @@ static int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits)
return 0;
}
+EXPORT_SYMBOL_GPL(vli_cmp);
/* Computes result = in << c, returning carry. Can modify in place
* (if result == in). 0 < shift < 64.
*/
-static u64 vli_lshift(u64 *result, const u64 *in, unsigned int shift,
- unsigned int ndigits)
+u64 vli_lshift(u64 *result, const u64 *in, unsigned int shift,
+ unsigned int ndigits)
{
u64 carry = 0;
int i;
@@ -277,9 +289,10 @@ static u64 vli_lshift(u64 *result, const u64 *in, unsigned int shift,
return carry;
}
+EXPORT_SYMBOL_GPL(vli_lshift);
/* Computes vli = vli >> 1. */
-static void vli_rshift1(u64 *vli, unsigned int ndigits)
+void vli_rshift1(u64 *vli, unsigned int ndigits)
{
u64 *end = vli;
u64 carry = 0;
@@ -292,10 +305,11 @@ static void vli_rshift1(u64 *vli, unsigned int ndigits)
carry = temp << 63;
}
}
+EXPORT_SYMBOL_GPL(vli_rshift1);
/* Computes result = left + right, returning carry. Can modify in place. */
-static u64 vli_add(u64 *result, const u64 *left, const u64 *right,
- unsigned int ndigits)
+u64 vli_add(u64 *result, const u64 *left, const u64 *right,
+ unsigned int ndigits)
{
u64 carry = 0;
int i;
@@ -312,10 +326,11 @@ static u64 vli_add(u64 *result, const u64 *left, const u64 *right,
return carry;
}
+EXPORT_SYMBOL_GPL(vli_add);
/* Computes result = left - right, returning borrow. Can modify in place. */
-static u64 vli_sub(u64 *result, const u64 *left, const u64 *right,
- unsigned int ndigits)
+u64 vli_sub(u64 *result, const u64 *left, const u64 *right,
+ unsigned int ndigits)
{
u64 borrow = 0;
int i;
@@ -332,6 +347,7 @@ static u64 vli_sub(u64 *result, const u64 *left, const u64 *right,
return borrow;
}
+EXPORT_SYMBOL_GPL(vli_sub);
static uint128_t mul_64_64(u64 left, u64 right)
{
@@ -368,8 +384,8 @@ static uint128_t add_128_128(uint128_t a, uint128_t b)
return result;
}
-static void vli_mult(u64 *result, const u64 *left, const u64 *right,
- unsigned int ndigits)
+void vli_mult(u64 *result, const u64 *left, const u64 *right,
+ unsigned int ndigits)
{
uint128_t r01 = { 0, 0 };
u64 r2 = 0;
@@ -403,8 +419,9 @@ static void vli_mult(u64 *result, const u64 *left, const u64 *right,
result[ndigits * 2 - 1] = r01.m_low;
}
+EXPORT_SYMBOL_GPL(vli_mult);
-static void vli_square(u64 *result, const u64 *left, unsigned int ndigits)
+void vli_square(u64 *result, const u64 *left, unsigned int ndigits)
{
uint128_t r01 = { 0, 0 };
u64 r2 = 0;
@@ -442,12 +459,13 @@ static void vli_square(u64 *result, const u64 *left, unsigned int ndigits)
result[ndigits * 2 - 1] = r01.m_low;
}
+EXPORT_SYMBOL_GPL(vli_square);
/* Computes result = (left + right) % mod.
* Assumes that left < mod and right < mod, result != mod.
*/
-static void vli_mod_add(u64 *result, const u64 *left, const u64 *right,
- const u64 *mod, unsigned int ndigits)
+void vli_mod_add(u64 *result, const u64 *left, const u64 *right,
+ const u64 *mod, unsigned int ndigits)
{
u64 carry;
@@ -459,12 +477,13 @@ static void vli_mod_add(u64 *result, const u64 *left, const u64 *right,
if (carry || vli_cmp(result, mod, ndigits) >= 0)
vli_sub(result, result, mod, ndigits);
}
+EXPORT_SYMBOL_GPL(vli_mod_add);
/* Computes result = (left - right) % mod.
* Assumes that left < mod and right < mod, result != mod.
*/
-static void vli_mod_sub(u64 *result, const u64 *left, const u64 *right,
- const u64 *mod, unsigned int ndigits)
+void vli_mod_sub(u64 *result, const u64 *left, const u64 *right,
+ const u64 *mod, unsigned int ndigits)
{
u64 borrow = vli_sub(result, left, right, ndigits);
@@ -475,6 +494,7 @@ static void vli_mod_sub(u64 *result, const u64 *left, const u64 *right,
if (borrow)
vli_add(result, result, mod, ndigits);
}
+EXPORT_SYMBOL_GPL(vli_mod_sub);
/* Computes result = input % mod.
* Assumes that input < mod, result != mod.
@@ -487,6 +507,7 @@ void vli_mod(u64 *result, const u64 *input, const u64 *mod,
else
vli_set(result, input, ndigits);
}
+EXPORT_SYMBOL_GPL(vli_mod);
/* Print vli in big-endian format.
* The bytes are printed in hex.
@@ -507,6 +528,7 @@ void vli_print(char *vli_name, const u64 *vli, unsigned int ndigits)
pr_info("%20s(BigEnd)=%s\n", vli_name, buf);
}
+EXPORT_SYMBOL_GPL(vli_print);
/* Computes result = (left * right) % mod.
* Assumes that left < mod and right < mod, result != mod.
@@ -552,6 +574,7 @@ void vli_mod_mult(u64 *result, const u64 *left, const u64 *right,
vli_sub(aa, aa, t1, ndigits);
}
}
+EXPORT_SYMBOL_GPL(vli_mod_mult);
/* Computes p_result = p_product % curve_p.
* See algorithm 5 and 6 from
@@ -663,8 +686,8 @@ static void vli_mmod_fast_256(u64 *result, const u64 *product,
/* Computes result = product % curve_prime
* from http://www.nsa.gov/ia/_files/nist-routines.pdf
*/
-static bool vli_mmod_fast(u64 *result, u64 *product,
- const u64 *curve_prime, unsigned int ndigits)
+bool vli_mmod_fast(u64 *result, u64 *product,
+ const u64 *curve_prime, unsigned int ndigits)
{
u64 tmp[2 * ndigits];
@@ -682,34 +705,37 @@ static bool vli_mmod_fast(u64 *result, u64 *product,
return true;
}
+EXPORT_SYMBOL_GPL(vli_mmod_fast);
/* Computes result = (left * right) % curve_prime. */
-static void vli_mod_mult_fast(u64 *result, const u64 *left, const u64 *right,
- const u64 *curve_prime, unsigned int ndigits)
+void vli_mod_mult_fast(u64 *result, const u64 *left, const u64 *right,
+ const u64 *curve_prime, unsigned int ndigits)
{
u64 product[2 * ndigits];
vli_mult(product, left, right, ndigits);
vli_mmod_fast(result, product, curve_prime, ndigits);
}
+EXPORT_SYMBOL_GPL(vli_mod_mult_fast);
/* Computes result = left^2 % curve_prime. */
-static void vli_mod_square_fast(u64 *result, const u64 *left,
- const u64 *curve_prime, unsigned int ndigits)
+void vli_mod_square_fast(u64 *result, const u64 *left,
+ const u64 *curve_prime, unsigned int ndigits)
{
u64 product[2 * ndigits];
vli_square(product, left, ndigits);
vli_mmod_fast(result, product, curve_prime, ndigits);
}
+EXPORT_SYMBOL_GPL(vli_mod_square_fast);
#define EVEN(vli) (!(vli[0] & 1))
/* Computes result = (1 / p_input) % mod. All VLIs are the same size.
* See "From Euclid's GCD to Montgomery Multiplication to the Great Divide"
* https://labs.oracle.com/techrep/2001/smli_tr-2001-95.pdf
*/
-static void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod,
- unsigned int ndigits)
+void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod,
+ unsigned int ndigits)
{
u64 a[ndigits], b[ndigits];
u64 u[ndigits], v[ndigits];
@@ -781,23 +807,25 @@ static void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod,
vli_set(result, u, ndigits);
}
+EXPORT_SYMBOL_GPL(vli_mod_inv);
/* ------ Point operations ------ */
/* Returns true if p_point is the point at infinity, false otherwise. */
-static bool ecc_point_is_zero(const struct ecc_point *point)
+bool ecc_point_is_zero(const struct ecc_point *point)
{
return (vli_is_zero(point->x, point->ndigits) &&
vli_is_zero(point->y, point->ndigits));
}
+EXPORT_SYMBOL_GPL(ecc_point_is_zero);
/* Point multiplication algorithm using Montgomery's ladder with co-Z
* coordinates. From http://eprint.iacr.org/2011/338.pdf
*/
/* Double in place */
-static void ecc_point_double_jacobian(u64 *x1, u64 *y1, u64 *z1,
- u64 *curve_prime, unsigned int ndigits)
+void ecc_point_double_jacobian(u64 *x1, u64 *y1, u64 *z1,
+ u64 *curve_prime, unsigned int ndigits)
{
/* t1 = x, t2 = y, t3 = z */
u64 t4[ndigits];
@@ -857,6 +885,7 @@ static void ecc_point_double_jacobian(u64 *x1, u64 *y1, u64 *z1,
vli_set(z1, y1, ndigits);
vli_set(y1, t4, ndigits);
}
+EXPORT_SYMBOL_GPL(ecc_point_double_jacobian);
/* Modify (x1, y1) => (x1 * z^2, y1 * z^3) */
static void apply_z(u64 *x1, u64 *y1, u64 *z, u64 *curve_prime,
@@ -1045,11 +1074,12 @@ void ecc_point_add(u64 *x1, u64 *y1, u64 *x2, u64 *y2, u64 *curve_prime,
vli_mod_mult_fast(t5, t5, t7, curve_prime, ndigits);
vli_set(x2, t5, ndigits);
}
+EXPORT_SYMBOL_GPL(ecc_point_add);
-static void ecc_point_mult(struct ecc_point *result,
- const struct ecc_point *point, const u64 *scalar,
- u64 *initial_z, u64 *curve_prime,
- unsigned int ndigits)
+void ecc_point_mult(struct ecc_point *result,
+ const struct ecc_point *point, const u64 *scalar,
+ u64 *initial_z, u64 *curve_prime,
+ unsigned int ndigits)
{
/* R0 and R1 */
u64 rx[2][ndigits];
@@ -1100,15 +1130,16 @@ static void ecc_point_mult(struct ecc_point *result,
vli_set(result->x, rx[0], ndigits);
vli_set(result->y, ry[0], ndigits);
}
+EXPORT_SYMBOL_GPL(ecc_point_mult);
-static inline void ecc_swap_digits(const u64 *in, u64 *out,
- unsigned int ndigits)
+void ecc_swap_digits(const u64 *in, u64 *out, unsigned int ndigits)
{
int i;
for (i = 0; i < ndigits; i++)
out[i] = __swab64(in[ndigits - 1 - i]);
}
+EXPORT_SYMBOL_GPL(ecc_swap_digits);
int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
const u8 *private_key, unsigned int private_key_len)
@@ -1133,6 +1164,7 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
return 0;
}
+EXPORT_SYMBOL_GPL(ecc_is_key_valid);
int ecc_is_pub_key_valid(unsigned int curve_id, unsigned int ndigits,
const u8 *pub_key, unsigned int pub_key_len)
@@ -1154,3 +1186,4 @@ int ecc_is_pub_key_valid(unsigned int curve_id, unsigned int ndigits,
return 0;
}
+EXPORT_SYMBOL_GPL(ecc_is_pub_key_valid);
diff --git a/crypto/ecc.h b/crypto/ecc.h
index 40f930d54c29..f947bf9a9d1e 100644
--- a/crypto/ecc.h
+++ b/crypto/ecc.h
@@ -31,6 +31,59 @@
#include "ecc_curve_defs.h"
+const struct ecc_curve *ecc_get_curve(unsigned int curve_id);
+struct ecc_point *ecc_alloc_point(unsigned int ndigits);
+void ecc_free_point(struct ecc_point *p);
+
+void vli_clear(u64 *vli, unsigned int ndigits);
+bool vli_is_zero(const u64 *vli, unsigned int ndigits);
+unsigned int vli_num_digits(const u64 *vli, unsigned int ndigits);
+unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits);
+void vli_set(u64 *dest, const u64 *src, unsigned int ndigits);
+void vli_copy_to_buf(u8 *dst_buf, unsigned int buf_len,
+ const u64 *src_vli, unsigned int ndigits);
+void vli_copy_from_buf(u64 *dst_vli, unsigned int ndigits,
+ const u8 *src_buf, unsigned int buf_len);
+int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits);
+u64 vli_lshift(u64 *result, const u64 *in, unsigned int shift,
+ unsigned int ndigits);
+void vli_rshift1(u64 *vli, unsigned int ndigits);
+u64 vli_add(u64 *result, const u64 *left, const u64 *right,
+ unsigned int ndigits);
+u64 vli_sub(u64 *result, const u64 *left, const u64 *right,
+ unsigned int ndigits);
+void vli_mult(u64 *result, const u64 *left, const u64 *right,
+ unsigned int ndigits);
+void vli_square(u64 *result, const u64 *left, unsigned int ndigits);
+void vli_mod_add(u64 *result, const u64 *left, const u64 *right,
+ const u64 *mod, unsigned int ndigits);
+void vli_mod_sub(u64 *result, const u64 *left, const u64 *right,
+ const u64 *mod, unsigned int ndigits);
+void vli_mod(u64 *result, const u64 *input, const u64 *mod,
+ unsigned int ndigits);
+void vli_print(char *vli_name, const u64 *vli, unsigned int ndigits);
+void vli_mod_mult(u64 *result, const u64 *left, const u64 *right,
+ const u64 *mod, unsigned int ndigits);
+bool vli_mmod_fast(u64 *result, u64 *product,
+ const u64 *curve_prime, unsigned int ndigits);
+void vli_mod_mult_fast(u64 *result, const u64 *left, const u64 *right,
+ const u64 *curve_prime, unsigned int ndigits);
+void vli_mod_square_fast(u64 *result, const u64 *left,
+ const u64 *curve_prime, unsigned int ndigits);
+void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod,
+ unsigned int ndigits);
+
+bool ecc_point_is_zero(const struct ecc_point *point);
+void ecc_point_double_jacobian(u64 *x1, u64 *y1, u64 *z1,
+ u64 *curve_prime, unsigned int ndigits);
+void ecc_point_add(u64 *x1, u64 *y1, u64 *x2, u64 *y2, u64 *curve_prime,
+ unsigned int ndigits);
+void ecc_point_mult(struct ecc_point *result,
+ const struct ecc_point *point, const u64 *scalar,
+ u64 *initial_z, u64 *curve_prime,
+ unsigned int ndigits);
+void ecc_swap_digits(const u64 *in, u64 *out, unsigned int ndigits);
+
/**
* ecc_is_key_valid() - Validate a given ECC private key
*
@@ -43,4 +96,7 @@
*/
int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
const u8 *private_key, unsigned int private_key_len);
+int ecc_is_pub_key_valid(unsigned int curve_id, unsigned int ndigits,
+ const u8 *pub_key, unsigned int pub_key_len);
+
#endif /* _CRYPTO_ECC_H */
--
1.7.6.3
^ permalink raw reply related
* [PATCH v2 2/6] crypto: ecc: add vli and ecc ops
From: Nitin Kumbhar @ 2017-02-03 11:12 UTC (permalink / raw)
To: herbert, davem; +Cc: linux-crypto, Nitin Kumbhar
In-Reply-To: <1486120375-13070-1-git-send-email-nkumbhar@nvidia.com>
Add functions to copy vli from buffers, to print vli in
big endian format, for vli mod and mod multiplication ops,
ecc point addition and ecc pub key validation.
Signed-off-by: Nitin Kumbhar <nkumbhar@nvidia.com>
---
crypto/ecc.c | 189 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 189 insertions(+), 0 deletions(-)
diff --git a/crypto/ecc.c b/crypto/ecc.c
index a8c10e725138..1b8e8d248859 100644
--- a/crypto/ecc.c
+++ b/crypto/ecc.c
@@ -208,6 +208,42 @@ static void vli_set(u64 *dest, const u64 *src, unsigned int ndigits)
dest[i] = src[i];
}
+/* Copy from vli to buf.
+ * For buffers smaller than vli: copy only LSB nbytes from vli.
+ * For buffers larger than vli : fill up remaining buf with zeroes.
+ */
+void vli_copy_to_buf(u8 *dst_buf, unsigned int buf_len,
+ const u64 *src_vli, unsigned int ndigits)
+{
+ unsigned int nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+ u8 *vli = (u8 *)src_vli;
+ int i;
+
+ for (i = 0; i < buf_len && i < nbytes; i++)
+ dst_buf[i] = vli[i];
+
+ for (; i < buf_len; i++)
+ dst_buf[i] = 0;
+}
+
+/* Copy from buffer to vli.
+ * For buffers smaller than vli: fill up remaining vli with zeroes.
+ * For buffers larger than vli : copy only LSB nbytes to vli.
+ */
+void vli_copy_from_buf(u64 *dst_vli, unsigned int ndigits,
+ const u8 *src_buf, unsigned int buf_len)
+{
+ unsigned int nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+ u8 *vli = (u8 *)dst_vli;
+ int i;
+
+ for (i = 0; i < buf_len && i < nbytes; i++)
+ vli[i] = src_buf[i];
+
+ for (; i < nbytes; i++)
+ vli[i] = 0;
+}
+
/* Returns sign of left - right. */
static int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits)
{
@@ -440,6 +476,83 @@ static void vli_mod_sub(u64 *result, const u64 *left, const u64 *right,
vli_add(result, result, mod, ndigits);
}
+/* Computes result = input % mod.
+ * Assumes that input < mod, result != mod.
+ */
+void vli_mod(u64 *result, const u64 *input, const u64 *mod,
+ unsigned int ndigits)
+{
+ if (vli_cmp(input, mod, ndigits) >= 0)
+ vli_sub(result, input, mod, ndigits);
+ else
+ vli_set(result, input, ndigits);
+}
+
+/* Print vli in big-endian format.
+ * The bytes are printed in hex.
+ */
+void vli_print(char *vli_name, const u64 *vli, unsigned int ndigits)
+{
+ int nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+ int buf_size = 2 * ECC_MAX_DIGIT_BYTES + 1;
+ unsigned char *c, buf[buf_size];
+ int i, j;
+
+ c = (unsigned char *)vli;
+
+ for (i = nbytes - 1, j = 0; i >= 0 && j < buf_size; i--, j += 2)
+ snprintf(&buf[j], 3, "%02x", *(c + i));
+
+ buf[j] = '\0';
+
+ pr_info("%20s(BigEnd)=%s\n", vli_name, buf);
+}
+
+/* Computes result = (left * right) % mod.
+ * Assumes that left < mod and right < mod, result != mod.
+ * Uses:
+ * (a * b) % m = ((a % m) * (b % m)) % m
+ * (a * b) % m = (a + a + ... + a) % m = b modular additions of (a % m)
+ */
+void vli_mod_mult(u64 *result, const u64 *left, const u64 *right,
+ const u64 *mod, unsigned int ndigits)
+{
+ u64 t1[ndigits], mm[ndigits];
+ u64 aa[ndigits], bb[ndigits];
+
+ vli_clear(result, ndigits);
+ vli_set(aa, left, ndigits);
+ vli_set(bb, right, ndigits);
+ vli_set(mm, mod, ndigits);
+
+ /* aa = aa % mm */
+ vli_mod(aa, aa, mm, ndigits);
+
+ /* bb = bb % mm */
+ vli_mod(bb, bb, mm, ndigits);
+
+ while (!vli_is_zero(bb, ndigits)) {
+
+ /* if bb is odd i.e. 0th bit set then add
+ * aa i.e. result = (result + aa) % mm
+ */
+ if (vli_test_bit(bb, 0))
+ vli_mod_add(result, result, aa, mm, ndigits);
+
+ /* bb = bb / 2 = bb >> 1 */
+ vli_rshift1(bb, ndigits);
+
+ /* aa = (aa * 2) % mm */
+ vli_sub(t1, mm, aa, ndigits);
+ if (vli_cmp(aa, t1, ndigits) == -1)
+ /* if aa < t1 then aa = aa * 2 = aa << 1*/
+ vli_lshift(aa, aa, 1, ndigits);
+ else
+ /* if aa >= t1 then aa = aa - t1 */
+ vli_sub(aa, aa, t1, ndigits);
+ }
+}
+
/* Computes p_result = p_product % curve_p.
* See algorithm 5 and 6 from
* http://www.isys.uni-klu.ac.at/PDF/2001-0126-MT.pdf
@@ -878,6 +991,61 @@ static void xycz_add_c(u64 *x1, u64 *y1, u64 *x2, u64 *y2, u64 *curve_prime,
vli_set(x1, t7, ndigits);
}
+/* Point addition.
+ * Add 2 distinct points on elliptic curve to get a new point.
+ *
+ * P = (x1,y1)and Q = (x2, y2) then P + Q = (x3,y3) where
+ * x3 = ((y2-y1)/(x2-x1))^2 - x1 - x2
+ * y3 = ((y2-y1)/(x2-x1))(x1-x3) - y1
+ *
+ * Q => P + Q
+ */
+void ecc_point_add(u64 *x1, u64 *y1, u64 *x2, u64 *y2, u64 *curve_prime,
+ unsigned int ndigits)
+{
+ /* t1 = X1, t2 = Y1, t3 = X2, t4 = Y2 */
+ u64 t5[ndigits];
+ u64 t6[ndigits];
+ u64 t7[ndigits];
+
+ /* t6 = x2 - x1 */
+ vli_mod_sub(t6, x2, x1, curve_prime, ndigits);
+ /* t6 = (x2 - x1)^2 = A */
+ vli_mod_square_fast(t6, t6, curve_prime, ndigits);
+ vli_mod_inv(t7, t6, curve_prime, ndigits);
+ /* t5 = x2 - x1 */
+ vli_mod_sub(t5, x2, x1, curve_prime, ndigits);
+ /* t5 = (x2 - x1)^2 = A */
+ vli_mod_square_fast(t5, t5, curve_prime, ndigits);
+ /* t1 = x1*A = B = x1*(x2-x1)^2*/
+ vli_mod_mult_fast(x1, x1, t5, curve_prime, ndigits);
+ /* t3 = x2*A = C = x2*(x2-x1)^2*/
+ vli_mod_mult_fast(x2, x2, t5, curve_prime, ndigits);
+ /* t4 = y2 - y1 */
+ vli_mod_sub(y2, y2, y1, curve_prime, ndigits);
+ /* t5 = (y2 - y1)^2 = D */
+ vli_mod_square_fast(t5, y2, curve_prime, ndigits);
+
+ /* t5 = D - B = (y2 - y1)^2 - x1*(x2-x1)^2 */
+ vli_mod_sub(t5, t5, x1, curve_prime, ndigits);
+ /* t5 = D - B - C = x3 = (y2 - y1)^2 - x1*(x2-x1)^2 - x2*(x2-x1)^2*/
+ vli_mod_sub(t5, t5, x2, curve_prime, ndigits);
+
+ /* t3 = C - B = x2*(x2-x1)^2 - x1*(x2-x1)^2 */
+ vli_mod_sub(x2, x2, x1, curve_prime, ndigits);
+ /* t2 = y1*(C - B) = y1*(x2*(x2-x1)^2 - x1*(x2-x1)^2)*/
+ vli_mod_mult_fast(y1, y1, x2, curve_prime, ndigits);
+ /* t3 = B - x3 = x1*(x2-x1)^2 - x3*/
+ vli_mod_sub(x2, x1, t5, curve_prime, ndigits);
+ /* t4 = (y2 - y1)*(B - x3) = (y2 - y1)*(x1*(x2-x1)^2 - x3)*/
+ vli_mod_mult_fast(y2, y2, x2, curve_prime, ndigits);
+ /* t4 = y3 = ((y2 - y1)*(x1*(x2-x1)^2 - x3)) - y1*/
+ vli_mod_sub(y2, y2, y1, curve_prime, ndigits);
+
+ vli_mod_mult_fast(t5, t5, t7, curve_prime, ndigits);
+ vli_set(x2, t5, ndigits);
+}
+
static void ecc_point_mult(struct ecc_point *result,
const struct ecc_point *point, const u64 *scalar,
u64 *initial_z, u64 *curve_prime,
@@ -965,3 +1133,24 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
return 0;
}
+
+int ecc_is_pub_key_valid(unsigned int curve_id, unsigned int ndigits,
+ const u8 *pub_key, unsigned int pub_key_len)
+{
+ const struct ecc_curve *curve = ecc_get_curve(curve_id);
+ int nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+ struct ecc_point p;
+
+ if (!pub_key || pub_key_len != 2 * nbytes)
+ return -EINVAL;
+
+ p.x = (u64 *)pub_key;
+ p.y = (u64 *)(pub_key + ECC_MAX_DIGIT_BYTES);
+ p.ndigits = ndigits;
+
+ if (vli_cmp(curve->p, p.x, ndigits) != 1 ||
+ vli_cmp(curve->p, p.y, ndigits) != 1)
+ return -EINVAL;
+
+ return 0;
+}
--
1.7.6.3
^ permalink raw reply related
* [PATCH v2 0/6] Add support for ECDSA algorithm
From: Nitin Kumbhar @ 2017-02-03 11:12 UTC (permalink / raw)
To: herbert, davem; +Cc: linux-crypto, Nitin Kumbhar
Hello,
This patch series adds support for Elliptic Curve Digital Signature
Algorithm (ECDSA). To reuse existing ECC functionality, which is
added as part of ECDH, it separates out ECC and ECDH so that
only ECC functionality is available for ECDSA even when ECDH is in
a disabled state.
Patch #1 restructures ECC and ECDH code such that ECC is not
dependent on ECDH config.
Patches #2 & #3 add vli and ecc functions which are required
for other Elliptic curve algorithms like ECDSA and ECIES.
Patch #4 adds support for ECDSA. This has been validated for P192
and P256 elliptic curves.
Patches #5 and #6 add ECDSA tests to validate ECDSA functionality
and measure ECDSA performance.
Changes in v2:
* Added ecc_is_pub_key_valid() for public key validation
* Use crypto_rng_get_bytes() to get random bytes
* Add documentation in ecdsa.h for ECDSA keys and related APIs
* Reorg ECDSA sign and verity tests to reuse code
Nitin Kumbhar (6):
crypto: ecc: separate out ecc and ecdh
crypto: ecc: add vli and ecc ops
crypto: ecc: export vli and ecc ops
crypto: ecdsa: add ECDSA SW implementation
crypto: testmgr: add ECDSA tests
crypto: tcrypt: add ECDSA test modes
crypto/Kconfig | 14 ++
crypto/Makefile | 8 +-
crypto/ecc.c | 421 +++++++++++++++++++++++++++++++-------------
crypto/ecc.h | 101 +++++++-----
crypto/ecc_curve_defs.h | 51 +-----
crypto/ecc_ecdh.h | 54 ++++++
crypto/ecdh.c | 4 +-
crypto/ecdh_helper.c | 94 ++++++++++
crypto/ecdsa.c | 362 ++++++++++++++++++++++++++++++++++++++
crypto/ecdsa_helper.c | 116 +++++++++++++
crypto/tcrypt.c | 250 ++++++++++++++++++++++++++-
crypto/tcrypt.h | 122 +++++++++++++
crypto/testmgr.c | 330 +++++++++++++++++++++++++++++++++++-
crypto/testmgr.h | 140 +++++++++++++++
include/crypto/akcipher.h | 5 +-
include/crypto/ecc.h | 24 +++
include/crypto/ecdh.h | 10 +-
include/crypto/ecdsa.h | 81 +++++++++
18 files changed, 1962 insertions(+), 225 deletions(-)
create mode 100644 crypto/ecc_ecdh.h
create mode 100644 crypto/ecdsa.c
create mode 100644 crypto/ecdsa_helper.c
create mode 100644 include/crypto/ecc.h
create mode 100644 include/crypto/ecdsa.h
--
1.7.6.3
^ permalink raw reply
* [PATCH v2 1/6] crypto: ecc: separate out ecc and ecdh
From: Nitin Kumbhar @ 2017-02-03 11:12 UTC (permalink / raw)
To: herbert, davem; +Cc: linux-crypto, Nitin Kumbhar
In-Reply-To: <1486120375-13070-1-git-send-email-nkumbhar@nvidia.com>
Add ECC operations i.e. vli and ecc point ops as a separate
module under CRYPTO_ECC kconfig. This allows other ECC
algorithms like ECDSA & ECIES to reuse these ops even when ECDH
is disabled with CRYPTO_ECDH kconfig.
With these changes, ECDH specific functions are consolidated
as ECDH helper routines and ECC curves are moved to ECC specific
files.
Signed-off-by: Nitin Kumbhar <nkumbhar@nvidia.com>
---
crypto/Kconfig | 7 +++
crypto/Makefile | 5 +-
crypto/ecc.c | 133 ++++++++++++++--------------------------------
crypto/ecc.h | 47 ++---------------
crypto/ecc_curve_defs.h | 51 ++++--------------
crypto/ecc_ecdh.h | 54 +++++++++++++++++++
crypto/ecdh.c | 4 +-
crypto/ecdh_helper.c | 94 +++++++++++++++++++++++++++++++++
include/crypto/ecc.h | 24 +++++++++
include/crypto/ecdh.h | 10 +---
10 files changed, 244 insertions(+), 185 deletions(-)
create mode 100644 crypto/ecc_ecdh.h
create mode 100644 include/crypto/ecc.h
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 160f08e721cc..e240075d6f46 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -127,9 +127,16 @@ config CRYPTO_DH
help
Generic implementation of the Diffie-Hellman algorithm.
+config CRYPTO_ECC
+ tristate "ECC functions"
+ help
+ Implementation of ECC functions
+
+
config CRYPTO_ECDH
tristate "ECDH algorithm"
select CRYTPO_KPP
+ select CRYPTO_ECC
help
Generic implementation of the ECDH algorithm
diff --git a/crypto/Makefile b/crypto/Makefile
index b8f0e3eb0791..827740a47a37 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -33,8 +33,9 @@ obj-$(CONFIG_CRYPTO_KPP2) += kpp.o
dh_generic-y := dh.o
dh_generic-y += dh_helper.o
obj-$(CONFIG_CRYPTO_DH) += dh_generic.o
-ecdh_generic-y := ecc.o
-ecdh_generic-y += ecdh.o
+
+obj-$(CONFIG_CRYPTO_ECC) += ecc.o
+ecdh_generic-y := ecdh.o
ecdh_generic-y += ecdh_helper.o
obj-$(CONFIG_CRYPTO_ECDH) += ecdh_generic.o
diff --git a/crypto/ecc.c b/crypto/ecc.c
index 414c78a9c214..a8c10e725138 100644
--- a/crypto/ecc.c
+++ b/crypto/ecc.c
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2013, Kenneth MacKay
* All rights reserved.
+ * Copyright (c) 2017, NVIDIA Corporation. All Rights Reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
@@ -28,16 +29,54 @@
#include <linux/slab.h>
#include <linux/swab.h>
#include <linux/fips.h>
-#include <crypto/ecdh.h>
#include "ecc.h"
-#include "ecc_curve_defs.h"
typedef struct {
u64 m_low;
u64 m_high;
} uint128_t;
+/* NIST P-192 */
+static u64 nist_p192_g_x[] = { 0xF4FF0AFD82FF1012ull, 0x7CBF20EB43A18800ull,
+ 0x188DA80EB03090F6ull };
+static u64 nist_p192_g_y[] = { 0x73F977A11E794811ull, 0x631011ED6B24CDD5ull,
+ 0x07192B95FFC8DA78ull };
+static u64 nist_p192_p[] = { 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFEull,
+ 0xFFFFFFFFFFFFFFFFull };
+static u64 nist_p192_n[] = { 0x146BC9B1B4D22831ull, 0xFFFFFFFF99DEF836ull,
+ 0xFFFFFFFFFFFFFFFFull };
+static struct ecc_curve nist_p192 = {
+ .name = "nist_192",
+ .g = {
+ .x = nist_p192_g_x,
+ .y = nist_p192_g_y,
+ .ndigits = 3,
+ },
+ .p = nist_p192_p,
+ .n = nist_p192_n
+};
+
+/* NIST P-256 */
+static u64 nist_p256_g_x[] = { 0xF4A13945D898C296ull, 0x77037D812DEB33A0ull,
+ 0xF8BCE6E563A440F2ull, 0x6B17D1F2E12C4247ull };
+static u64 nist_p256_g_y[] = { 0xCBB6406837BF51F5ull, 0x2BCE33576B315ECEull,
+ 0x8EE7EB4A7C0F9E16ull, 0x4FE342E2FE1A7F9Bull };
+static u64 nist_p256_p[] = { 0xFFFFFFFFFFFFFFFFull, 0x00000000FFFFFFFFull,
+ 0x0000000000000000ull, 0xFFFFFFFF00000001ull };
+static u64 nist_p256_n[] = { 0xF3B9CAC2FC632551ull, 0xBCE6FAADA7179E84ull,
+ 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFF00000000ull };
+static struct ecc_curve nist_p256 = {
+ .name = "nist_256",
+ .g = {
+ .x = nist_p256_g_x,
+ .y = nist_p256_g_y,
+ .ndigits = 4,
+ },
+ .p = nist_p256_p,
+ .n = nist_p256_n
+};
+
static inline const struct ecc_curve *ecc_get_curve(unsigned int curve_id)
{
switch (curve_id) {
@@ -926,93 +965,3 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
return 0;
}
-
-int ecdh_make_pub_key(unsigned int curve_id, unsigned int ndigits,
- const u8 *private_key, unsigned int private_key_len,
- u8 *public_key, unsigned int public_key_len)
-{
- int ret = 0;
- struct ecc_point *pk;
- u64 priv[ndigits];
- unsigned int nbytes;
- const struct ecc_curve *curve = ecc_get_curve(curve_id);
-
- if (!private_key || !curve) {
- ret = -EINVAL;
- goto out;
- }
-
- ecc_swap_digits((const u64 *)private_key, priv, ndigits);
-
- pk = ecc_alloc_point(ndigits);
- if (!pk) {
- ret = -ENOMEM;
- goto out;
- }
-
- ecc_point_mult(pk, &curve->g, priv, NULL, curve->p, ndigits);
- if (ecc_point_is_zero(pk)) {
- ret = -EAGAIN;
- goto err_free_point;
- }
-
- nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
- ecc_swap_digits(pk->x, (u64 *)public_key, ndigits);
- ecc_swap_digits(pk->y, (u64 *)&public_key[nbytes], ndigits);
-
-err_free_point:
- ecc_free_point(pk);
-out:
- return ret;
-}
-
-int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
- const u8 *private_key, unsigned int private_key_len,
- const u8 *public_key, unsigned int public_key_len,
- u8 *secret, unsigned int secret_len)
-{
- int ret = 0;
- struct ecc_point *product, *pk;
- u64 priv[ndigits];
- u64 rand_z[ndigits];
- unsigned int nbytes;
- const struct ecc_curve *curve = ecc_get_curve(curve_id);
-
- if (!private_key || !public_key || !curve) {
- ret = -EINVAL;
- goto out;
- }
-
- nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
-
- get_random_bytes(rand_z, nbytes);
-
- pk = ecc_alloc_point(ndigits);
- if (!pk) {
- ret = -ENOMEM;
- goto out;
- }
-
- product = ecc_alloc_point(ndigits);
- if (!product) {
- ret = -ENOMEM;
- goto err_alloc_product;
- }
-
- ecc_swap_digits((const u64 *)public_key, pk->x, ndigits);
- ecc_swap_digits((const u64 *)&public_key[nbytes], pk->y, ndigits);
- ecc_swap_digits((const u64 *)private_key, priv, ndigits);
-
- ecc_point_mult(product, pk, priv, rand_z, curve->p, ndigits);
-
- ecc_swap_digits(product->x, (u64 *)secret, ndigits);
-
- if (ecc_point_is_zero(product))
- ret = -EFAULT;
-
- ecc_free_point(product);
-err_alloc_product:
- ecc_free_point(pk);
-out:
- return ret;
-}
diff --git a/crypto/ecc.h b/crypto/ecc.h
index 663d598c7406..40f930d54c29 100644
--- a/crypto/ecc.h
+++ b/crypto/ecc.h
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2013, Kenneth MacKay
* All rights reserved.
+ * Copyright (c) 2017, NVIDIA Corporation. All Rights Reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
@@ -26,12 +27,12 @@
#ifndef _CRYPTO_ECC_H
#define _CRYPTO_ECC_H
-#define ECC_MAX_DIGITS 4 /* 256 */
+#include <crypto/ecc.h>
-#define ECC_DIGITS_TO_BYTES_SHIFT 3
+#include "ecc_curve_defs.h"
/**
- * ecc_is_key_valid() - Validate a given ECDH private key
+ * ecc_is_key_valid() - Validate a given ECC private key
*
* @curve_id: id representing the curve to use
* @ndigits: curve number of digits
@@ -42,42 +43,4 @@
*/
int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
const u8 *private_key, unsigned int private_key_len);
-
-/**
- * ecdh_make_pub_key() - Compute an ECC public key
- *
- * @curve_id: id representing the curve to use
- * @private_key: pregenerated private key for the given curve
- * @private_key_len: length of private_key
- * @public_key: buffer for storing the public key generated
- * @public_key_len: length of the public_key buffer
- *
- * Returns 0 if the public key was generated successfully, a negative value
- * if an error occurred.
- */
-int ecdh_make_pub_key(const unsigned int curve_id, unsigned int ndigits,
- const u8 *private_key, unsigned int private_key_len,
- u8 *public_key, unsigned int public_key_len);
-
-/**
- * crypto_ecdh_shared_secret() - Compute a shared secret
- *
- * @curve_id: id representing the curve to use
- * @private_key: private key of part A
- * @private_key_len: length of private_key
- * @public_key: public key of counterpart B
- * @public_key_len: length of public_key
- * @secret: buffer for storing the calculated shared secret
- * @secret_len: length of the secret buffer
- *
- * Note: It is recommended that you hash the result of crypto_ecdh_shared_secret
- * before using it for symmetric encryption or HMAC.
- *
- * Returns 0 if the shared secret was generated successfully, a negative value
- * if an error occurred.
- */
-int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
- const u8 *private_key, unsigned int private_key_len,
- const u8 *public_key, unsigned int public_key_len,
- u8 *secret, unsigned int secret_len);
-#endif
+#endif /* _CRYPTO_ECC_H */
diff --git a/crypto/ecc_curve_defs.h b/crypto/ecc_curve_defs.h
index 03ae5f714028..baacf32bca16 100644
--- a/crypto/ecc_curve_defs.h
+++ b/crypto/ecc_curve_defs.h
@@ -1,3 +1,12 @@
+/*
+ * Copyright (c) 2017, NVIDIA Corporation. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
#ifndef _CRYTO_ECC_CURVE_DEFS_H
#define _CRYTO_ECC_CURVE_DEFS_H
@@ -14,44 +23,4 @@ struct ecc_curve {
u64 *n;
};
-/* NIST P-192 */
-static u64 nist_p192_g_x[] = { 0xF4FF0AFD82FF1012ull, 0x7CBF20EB43A18800ull,
- 0x188DA80EB03090F6ull };
-static u64 nist_p192_g_y[] = { 0x73F977A11E794811ull, 0x631011ED6B24CDD5ull,
- 0x07192B95FFC8DA78ull };
-static u64 nist_p192_p[] = { 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFEull,
- 0xFFFFFFFFFFFFFFFFull };
-static u64 nist_p192_n[] = { 0x146BC9B1B4D22831ull, 0xFFFFFFFF99DEF836ull,
- 0xFFFFFFFFFFFFFFFFull };
-static struct ecc_curve nist_p192 = {
- .name = "nist_192",
- .g = {
- .x = nist_p192_g_x,
- .y = nist_p192_g_y,
- .ndigits = 3,
- },
- .p = nist_p192_p,
- .n = nist_p192_n
-};
-
-/* NIST P-256 */
-static u64 nist_p256_g_x[] = { 0xF4A13945D898C296ull, 0x77037D812DEB33A0ull,
- 0xF8BCE6E563A440F2ull, 0x6B17D1F2E12C4247ull };
-static u64 nist_p256_g_y[] = { 0xCBB6406837BF51F5ull, 0x2BCE33576B315ECEull,
- 0x8EE7EB4A7C0F9E16ull, 0x4FE342E2FE1A7F9Bull };
-static u64 nist_p256_p[] = { 0xFFFFFFFFFFFFFFFFull, 0x00000000FFFFFFFFull,
- 0x0000000000000000ull, 0xFFFFFFFF00000001ull };
-static u64 nist_p256_n[] = { 0xF3B9CAC2FC632551ull, 0xBCE6FAADA7179E84ull,
- 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFF00000000ull };
-static struct ecc_curve nist_p256 = {
- .name = "nist_256",
- .g = {
- .x = nist_p256_g_x,
- .y = nist_p256_g_y,
- .ndigits = 4,
- },
- .p = nist_p256_p,
- .n = nist_p256_n
-};
-
-#endif
+#endif /* _CRYTO_ECC_CURVE_DEFS_H */
diff --git a/crypto/ecc_ecdh.h b/crypto/ecc_ecdh.h
new file mode 100644
index 000000000000..f77b1fe094c9
--- /dev/null
+++ b/crypto/ecc_ecdh.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2013, Kenneth MacKay. All rights reserved.
+ * Copyright (c) 2017, NVIDIA Corporation. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#ifndef _CRYPTO_ECC_ECDH_H
+#define _CRYPTO_ECC_ECDH_H
+
+#include "ecc.h"
+
+/**
+ * ecdh_make_pub_key() - Compute an ECC public key
+ *
+ * @curve_id: id representing the curve to use
+ * @private_key: pregenerated private key for the given curve
+ * @private_key_len: length of private_key
+ * @public_key: buffer for storing the public key generated
+ * @public_key_len: length of the public_key buffer
+ *
+ * Returns 0 if the public key was generated successfully, a negative value
+ * if an error occurred.
+ */
+int ecdh_make_pub_key(const unsigned int curve_id, unsigned int ndigits,
+ const u8 *private_key, unsigned int private_key_len,
+ u8 *public_key, unsigned int public_key_len);
+
+/**
+ * crypto_ecdh_shared_secret() - Compute a shared secret
+ *
+ * @curve_id: id representing the curve to use
+ * @private_key: private key of part A
+ * @private_key_len: length of private_key
+ * @public_key: public key of counterpart B
+ * @public_key_len: length of public_key
+ * @secret: buffer for storing the calculated shared secret
+ * @secret_len: length of the secret buffer
+ *
+ * Note: It is recommended that you hash the result of crypto_ecdh_shared_secret
+ * before using it for symmetric encryption or HMAC.
+ *
+ * Returns 0 if the shared secret was generated successfully, a negative value
+ * if an error occurred.
+ */
+int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
+ const u8 *private_key, unsigned int private_key_len,
+ const u8 *public_key, unsigned int public_key_len,
+ u8 *secret, unsigned int secret_len);
+
+#endif /* _CRYPTO_ECC_ECDH_H */
diff --git a/crypto/ecdh.c b/crypto/ecdh.c
index 3de289806d67..2b83ff3a4b9a 100644
--- a/crypto/ecdh.c
+++ b/crypto/ecdh.c
@@ -2,6 +2,7 @@
*
* Copyright (c) 2016, Intel Corporation
* Authors: Salvator Benedetto <salvatore.benedetto@intel.com>
+ * Copyright (c) 2017, NVIDIA Corporation.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public Licence
@@ -14,7 +15,8 @@
#include <crypto/kpp.h>
#include <crypto/ecdh.h>
#include <linux/scatterlist.h>
-#include "ecc.h"
+
+#include "ecc_ecdh.h"
struct ecdh_ctx {
unsigned int curve_id;
diff --git a/crypto/ecdh_helper.c b/crypto/ecdh_helper.c
index 3cd8a2414e60..b3857f3bfcee 100644
--- a/crypto/ecdh_helper.c
+++ b/crypto/ecdh_helper.c
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2016, Intel Corporation
* Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
+ * Copyright (c) 2017, NVIDIA Corporation.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public Licence
@@ -11,9 +12,12 @@
#include <linux/export.h>
#include <linux/err.h>
#include <linux/string.h>
+#include <linux/random.h>
#include <crypto/ecdh.h>
#include <crypto/kpp.h>
+#include "ecc_ecdh.h"
+
#define ECDH_KPP_SECRET_MIN_SIZE (sizeof(struct kpp_secret) + 2 * sizeof(short))
static inline u8 *ecdh_pack_data(void *dst, const void *src, size_t sz)
@@ -28,6 +32,96 @@
return src + sz;
}
+int ecdh_make_pub_key(unsigned int curve_id, unsigned int ndigits,
+ const u8 *private_key, unsigned int private_key_len,
+ u8 *public_key, unsigned int public_key_len)
+{
+ int ret = 0;
+ struct ecc_point *pk;
+ u64 priv[ndigits];
+ unsigned int nbytes;
+ const struct ecc_curve *curve = ecc_get_curve(curve_id);
+
+ if (!private_key || !curve) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ ecc_swap_digits((const u64 *)private_key, priv, ndigits);
+
+ pk = ecc_alloc_point(ndigits);
+ if (!pk) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ ecc_point_mult(pk, &curve->g, priv, NULL, curve->p, ndigits);
+ if (ecc_point_is_zero(pk)) {
+ ret = -EAGAIN;
+ goto err_free_point;
+ }
+
+ nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+ ecc_swap_digits(pk->x, (u64 *)public_key, ndigits);
+ ecc_swap_digits(pk->y, (u64 *)&public_key[nbytes], ndigits);
+
+err_free_point:
+ ecc_free_point(pk);
+out:
+ return ret;
+}
+
+int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
+ const u8 *private_key, unsigned int private_key_len,
+ const u8 *public_key, unsigned int public_key_len,
+ u8 *secret, unsigned int secret_len)
+{
+ int ret = 0;
+ struct ecc_point *product, *pk;
+ u64 priv[ndigits];
+ u64 rand_z[ndigits];
+ unsigned int nbytes;
+ const struct ecc_curve *curve = ecc_get_curve(curve_id);
+
+ if (!private_key || !public_key || !curve) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+
+ get_random_bytes(rand_z, nbytes);
+
+ pk = ecc_alloc_point(ndigits);
+ if (!pk) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ product = ecc_alloc_point(ndigits);
+ if (!product) {
+ ret = -ENOMEM;
+ goto err_alloc_product;
+ }
+
+ ecc_swap_digits((const u64 *)public_key, pk->x, ndigits);
+ ecc_swap_digits((const u64 *)&public_key[nbytes], pk->y, ndigits);
+ ecc_swap_digits((const u64 *)private_key, priv, ndigits);
+
+ ecc_point_mult(product, pk, priv, rand_z, curve->p, ndigits);
+
+ ecc_swap_digits(product->x, (u64 *)secret, ndigits);
+
+ if (ecc_point_is_zero(product))
+ ret = -EFAULT;
+
+ ecc_free_point(product);
+err_alloc_product:
+ ecc_free_point(pk);
+out:
+ return ret;
+}
+
int crypto_ecdh_key_len(const struct ecdh *params)
{
return ECDH_KPP_SECRET_MIN_SIZE + params->key_size;
diff --git a/include/crypto/ecc.h b/include/crypto/ecc.h
new file mode 100644
index 000000000000..27957f805fd6
--- /dev/null
+++ b/include/crypto/ecc.h
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2017, NVIDIA Corporation. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+
+#ifndef _CRYPTO_ECC_
+#define _CRYPTO_ECC_
+
+/* Curves IDs */
+#define ECC_CURVE_NIST_P192 0x0001
+#define ECC_CURVE_NIST_P256 0x0002
+
+#define ECC_MAX_DIGITS 4 /* 256 */
+
+#define ECC_DIGITS_TO_BYTES_SHIFT 3
+
+#define ECC_MAX_DIGIT_BYTES (ECC_MAX_DIGITS << ECC_DIGITS_TO_BYTES_SHIFT)
+
+#endif /* _CRYPTO_ECC_ */
diff --git a/include/crypto/ecdh.h b/include/crypto/ecdh.h
index 03a64f62ba7a..c8556305acad 100644
--- a/include/crypto/ecdh.h
+++ b/include/crypto/ecdh.h
@@ -3,6 +3,7 @@
*
* Copyright (c) 2016, Intel Corporation
* Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
+ * Copyright (c) 2017, NVIDIA Corporation.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
@@ -13,24 +14,19 @@
#ifndef _CRYPTO_ECDH_
#define _CRYPTO_ECDH_
+#include <crypto/ecc.h>
+
/**
* DOC: ECDH Helper Functions
*
* To use ECDH with the KPP cipher API, the following data structure and
* functions should be used.
*
- * The ECC curves known to the ECDH implementation are specified in this
- * header file.
- *
* To use ECDH with KPP, the following functions should be used to operate on
* an ECDH private key. The packet private key that can be set with
* the KPP API function call of crypto_kpp_set_secret.
*/
-/* Curves IDs */
-#define ECC_CURVE_NIST_P192 0x0001
-#define ECC_CURVE_NIST_P256 0x0002
-
/**
* struct ecdh - define an ECDH private key
*
--
1.7.6.3
^ permalink raw reply related
* Re: [PATCH 3/6] async_tx: Handle DMA devices having support for fewer PQ coefficients
From: Anup Patel @ 2017-02-03 11:00 UTC (permalink / raw)
To: Dan Williams
Cc: Vinod Koul, Rob Herring, Mark Rutland, Herbert Xu,
David S . Miller, Jassi Brar, Ray Jui, Scott Branden, Jon Mason,
Rob Rice, BCM Kernel Feedback, dmaengine@vger.kernel.org,
Device Tree, linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, linux-crypto, linux-raid
In-Reply-To: <CAPcyv4gBFu7skx_8cvsaL0sL5=44DMcWY_EjWSCfaNd8oh=Svw@mail.gmail.com>
On Thu, Feb 2, 2017 at 11:31 AM, Dan Williams <dan.j.williams@intel.com> wrote:
>
> On Wed, Feb 1, 2017 at 8:47 PM, Anup Patel <anup.patel@broadcom.com> wrote:
> > The DMAENGINE framework assumes that if PQ offload is supported by a
> > DMA device then all 256 PQ coefficients are supported. This assumption
> > does not hold anymore because we now have BCM-SBA-RAID offload engine
> > which supports PQ offload with limited number of PQ coefficients.
> >
> > This patch extends async_tx APIs to handle DMA devices with support
> > for fewer PQ coefficients.
> >
> > Signed-off-by: Anup Patel <anup.patel@broadcom.com>
> > Reviewed-by: Scott Branden <scott.branden@broadcom.com>
> > ---
> > crypto/async_tx/async_pq.c | 3 +++
> > crypto/async_tx/async_raid6_recov.c | 12 ++++++++++--
> > include/linux/dmaengine.h | 19 +++++++++++++++++++
> > include/linux/raid/pq.h | 3 +++
> > 4 files changed, 35 insertions(+), 2 deletions(-)
>
> So, I hate the way async_tx does these checks on each operation, and
> it's ok for me to say that because it's my fault. Really it's md that
> should be validating engine offload capabilities once at the beginning
> of time. I'd rather we move in that direction than continue to pile
> onto a bad design.
Yes, indeed. All async_tx APIs have lot of checks and for high throughput
RAID offload engine these checks can add some overhead.
I think doing checks in Linux md would be certainly better but this would
mean lot of changes in Linux md as well as remove checks in async_tx.
Also, async_tx APIs should not find DMA channel on its own instead it
should rely on Linux md to provide DMA channel pointer as parameter.
It's better to do checks cleanup in async_tx as separate patchset and
keep this patchset simple.
Regards,
Anup
^ permalink raw reply
* Re: [PATCH 1/2] crypto: arm64/aes - don't use IV buffer to return final keystream block
From: Herbert Xu @ 2017-02-03 10:22 UTC (permalink / raw)
To: Ard Biesheuvel; +Cc: linux-crypto, linux-arm-kernel
In-Reply-To: <1486035536-895-1-git-send-email-ard.biesheuvel@linaro.org>
On Thu, Feb 02, 2017 at 11:38:55AM +0000, Ard Biesheuvel wrote:
> The arm64 bit sliced AES core code uses the IV buffer to pass the final
> keystream block back to the glue code if the input is not a multiple of
> the block size, so that the asm code does not have to deal with anything
> except 16 byte blocks. This is done under the assumption that the outgoing
> IV is meaningless anyway in this case, given that chaining is no longer
> possible under these circumstances.
>
> However, as it turns out, the CCM driver does expect the IV to retain
> a value that is equal to the original IV except for the counter value,
> and even interprets byte zero as a length indicator, which may result
> in memory corruption if the IV is overwritten with something else.
>
> So use a separate buffer to return the final keystream block.
>
> Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Patch applied. Thanks.
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply
* Re: [PATCH 2/2] crypto: arm/aes - don't use IV buffer to return final keystream block
From: Herbert Xu @ 2017-02-03 10:22 UTC (permalink / raw)
To: Ard Biesheuvel; +Cc: linux-crypto, linux-arm-kernel
In-Reply-To: <1486035536-895-2-git-send-email-ard.biesheuvel@linaro.org>
On Thu, Feb 02, 2017 at 11:38:56AM +0000, Ard Biesheuvel wrote:
> The ARM bit sliced AES core code uses the IV buffer to pass the final
> keystream block back to the glue code if the input is not a multiple of
> the block size, so that the asm code does not have to deal with anything
> except 16 byte blocks. This is done under the assumption that the outgoing
> IV is meaningless anyway in this case, given that chaining is no longer
> possible under these circumstances.
>
> However, as it turns out, the CCM driver does expect the IV to retain
> a value that is equal to the original IV except for the counter value,
> and even interprets byte zero as a length indicator, which may result
> in memory corruption if the IV is overwritten with something else.
>
> So use a separate buffer to return the final keystream block.
>
> Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Patch applied. Thanks.
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply
* Re: [PATCH v3 00/10] crypto - AES for ARM/arm64 updates for v4.11 (round #2)
From: Herbert Xu @ 2017-02-03 10:22 UTC (permalink / raw)
To: Ard Biesheuvel; +Cc: linux-crypto, linux-arm-kernel
In-Reply-To: <1485645939-17126-1-git-send-email-ard.biesheuvel@linaro.org>
On Sat, Jan 28, 2017 at 11:25:29PM +0000, Ard Biesheuvel wrote:
> Patch #1 is a fix for the CBC chaining issue that was discussed on the
> mailing list. The driver itself is queued for v4.11, so this fix can go
> right on top.
>
> Patches #2 - #6 clear the cra_alignmasks of various drivers: all NEON
> capable CPUs can perform unaligned accesses, and the advantage of using
> the slightly faster aligned accessors (which only exist on ARM not arm64)
> is certainly outweighed by the cost of copying data to suitably aligned
> buffers.
>
> NOTE: patch #5 won't apply unless 'crypto: arm64/aes-blk - honour iv_out
> requirement in CBC and CTR modes' is applied first, which was sent out
> separately as a bugfix for v3.16 - v4.9. If this is a problem, this patch
> can wait.
>
> Patch #7 and #8 are minor tweaks to the new scalar AES code.
>
> Patch #9 improves the performance of the plain NEON AES code, to make it
> more suitable as a fallback for the new bitsliced NEON code, which can
> only operate on 8 blocks in parallel, and needs another driver to perform
> CBC encryption or XTS tweak generation.
>
> Patch #10 updates the new bitsliced AES NEON code to switch to the plain
> NEON driver as a fallback.
>
> Patches #9 and #10 improve the performance of CBC encryption by ~35% on
> low end cores such as the Cortex-A53 that can be found in the Raspberry Pi3
>
> Changes since v2:
> - use polynomial multiply NEON instruction for multiplication by x^2, this
> eliminates 4 instructions from the decrypt path (#9)
>
> Changes since v1:
> - shave off another few cycles from the sequential AES NEON code (patch #9)
Patches 2-10 applied.
Thanks,
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply
* Re: [PATCH] crypto: ccp: Fix double add when creating new DMA command
From: Herbert Xu @ 2017-02-03 10:21 UTC (permalink / raw)
To: Gary R Hook; +Cc: linux-crypto, thomas.lendacky, davem
In-Reply-To: <20170127230904.31399.58184.stgit@taos>
On Fri, Jan 27, 2017 at 05:09:04PM -0600, Gary R Hook wrote:
> Eliminate a double-add by creating a new list to manage
> command descriptors when created; move the descriptor to
> the pending list when the command is submitted. This
>
>
> Signed-off-by: Gary R Hook <gary.hook@amd.com>
Patch applied. Thanks.
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply
* Re: [PATCH] crypto: ccp: Fix DMA operations when IOMMU is enabled
From: Herbert Xu @ 2017-02-03 10:21 UTC (permalink / raw)
To: Gary R Hook; +Cc: linux-crypto, thomas.lendacky, davem
In-Reply-To: <20170127212845.27618.32169.stgit@taos>
On Fri, Jan 27, 2017 at 03:28:45PM -0600, Gary R Hook wrote:
> An I/O page fault occurs when the IOMMU is enabled on a
> system that supports the v5 CCP. DMA operations use a
> Request ID value that does not match what is expected by
> the IOMMU, resulting in the I/O page fault. Setting the
> Request ID value to 0 corrects this issue.
>
> Signed-off-by: Gary R Hook <gary.hook@amd.com>
Patch applied. Thanks.
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply
* Re: [PATCH 0/8] Bug fixes
From: Herbert Xu @ 2017-02-03 10:21 UTC (permalink / raw)
To: Harsh Jain; +Cc: linux-crypto, hariprasad, netdev, arjun, atul.gupta
In-Reply-To: <cover.1485501428.git.harsh@chelsio.com>
On Fri, Jan 27, 2017 at 04:09:04PM +0530, Harsh Jain wrote:
> This patch series is based on Herbert's cryptodev-2.6 tree and depends on
> patch series "Bug Fixes for 4.10". It includes Bug Fixes.
>
> Atul Gupta (2)
> crypto:chcr-Change flow IDs
> crypto:chcr- Fix wrong typecasting
> Harsh Jain (8):
> crypto:chcr- Fix key length for RFC4106
> crypto:chcr-fix itnull.cocci warnings
> crypto:chcr- Use cipher instead of Block Cipher in gcm setkey
> crypto:chcr: Change cra_flags for cipher algos
> crypto:chcr- Change algo priority
> crypto:chcr-Fix Smatch Complaint
All applied. Thanks.
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply
* Re: [PATCH v3 00/12] crypto: atmel-authenc: add support to authenc(hmac(shaX),Y(aes)) modes
From: Herbert Xu @ 2017-02-03 10:20 UTC (permalink / raw)
To: Cyrille Pitchen
Cc: smueller, nicolas.ferre, linux-kernel, linux-crypto, davem,
linux-arm-kernel
In-Reply-To: <cover.1485443478.git.cyrille.pitchen@atmel.com>
On Thu, Jan 26, 2017 at 05:07:45PM +0100, Cyrille Pitchen wrote:
> Hi all,
>
> this series of patches has been based and tested on next-20170125 with
> CRYPTO_MANAGER_DISABLED_TESTS not set.
>
> The series adds support to the hmac(shaX) algorithms first, then combines
> both the Atmel SHA and AES hardware accelerators to implement
> authenc(hmac(shaX),Y(aes)) algorithms as used by IPSEC/SSL connections.
>
> It has also been tested with strongswan + xl2tpd to create an IPSEC+L2TP
> (transport mode) VPN and strongswan only (tunnel mode) for an IPSEC VPN.
>
> Then iperf was used to measure the bandwidth improvement in tunnel mode:
>
> drivers AES SHA SPLIP iperf half-duplex
> Mbit/s
> authenc(hmac(sha1-generic),cbc(aes)) SW SW N/A 27.7
> authenc(hmac(sha1-generic),atmel-cbc-aes) HW SW N/A 30.2 (mainline)
> authenc(atmel-hmac-sha1,atmel-cbc-aes) HW HW no 29.1
> atmel-authenc-hmac-sha1-cbc-aes HW HW yes 38.8
>
> SPLIP: Secure Protocol Layers Improved Performances (AES+SHA combined).
>
> Some patches of this series are purely transitional: I've split the
> modifications into many patches to ease the review.
All applied. Thanks.
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply
* Re: [PATCHv2] crypto: doc - Fix hash export state information
From: Herbert Xu @ 2017-02-03 10:20 UTC (permalink / raw)
To: Rabin Vincent; +Cc: linux-crypto, smueller, Rabin Vincent
In-Reply-To: <1485444780-17201-1-git-send-email-rabin.vincent@axis.com>
On Thu, Jan 26, 2017 at 04:33:00PM +0100, Rabin Vincent wrote:
> From: Rabin Vincent <rabinv@axis.com>
>
> The documentation states that crypto_ahash_reqsize() provides the size
> of the state structure used by crypto_ahash_export(). But it's actually
> crypto_ahash_statesize() which provides this size.
>
> Signed-off-by: Rabin Vincent <rabinv@axis.com>
Patch applied. Thanks.
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply
* Re: [PATCH v2 0/2 ] Bug Fixes for 4.10
From: Herbert Xu @ 2017-02-03 10:20 UTC (permalink / raw)
To: Harsh Jain; +Cc: linux-crypto
In-Reply-To: <cover.1485233822.git.harsh@chelsio.com>
On Tue, Jan 24, 2017 at 10:34:31AM +0530, Harsh Jain wrote:
> This patch series includes critical bug fixes
>
> Atul Gupta (2):
> crypto:chcr- Fix panic on dma_unmap_sg
> crypto:chcr- Check device is allocated before use
All applied. I also added the key length check as I think it can
be exploited through algif_aead.
Thanks,
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply
* Re: [PATCH v2 2/4] crypto: ccm - switch to separate cbcmac driver
From: Ard Biesheuvel @ 2017-02-02 18:56 UTC (permalink / raw)
To: linux-crypto@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org, Herbert Xu, Ard Biesheuvel
In-Reply-To: <1485646413-17491-3-git-send-email-ard.biesheuvel@linaro.org>
On 28 January 2017 at 23:33, Ard Biesheuvel <ard.biesheuvel@linaro.org> wrote:
> Update the generic CCM driver to defer CBC-MAC processing to a
> dedicated CBC-MAC ahash transform rather than open coding this
> transform (and much of the associated scatterwalk plumbing) in
> the CCM driver itself.
>
> This cleans up the code considerably, but more importantly, it allows
> the use of alternative CBC-MAC implementations that don't suffer from
> performance degradation due to significant setup time (e.g., the NEON
> based AES code needs to load the entire S-box into SIMD registers, which
> cannot be amortized over the entire input when using the AES cipher
> directly)
>
> Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
I need to respin this: the idata[] buffer in crypto_ccm_auth() should
be 16 bytes not 6
> ---
> crypto/Kconfig | 1 +
> crypto/ccm.c | 376 +++++++++++++-------
> 2 files changed, 240 insertions(+), 137 deletions(-)
>
> diff --git a/crypto/Kconfig b/crypto/Kconfig
> index 160f08e721cc..e8269d1b0282 100644
> --- a/crypto/Kconfig
> +++ b/crypto/Kconfig
> @@ -263,6 +263,7 @@ comment "Authenticated Encryption with Associated Data"
> config CRYPTO_CCM
> tristate "CCM support"
> select CRYPTO_CTR
> + select CRYPTO_HASH
> select CRYPTO_AEAD
> help
> Support for Counter with CBC MAC. Required for IPsec.
> diff --git a/crypto/ccm.c b/crypto/ccm.c
> index 26b924d1e582..62dbaa58eeb0 100644
> --- a/crypto/ccm.c
> +++ b/crypto/ccm.c
> @@ -11,6 +11,7 @@
> */
>
> #include <crypto/internal/aead.h>
> +#include <crypto/internal/hash.h>
> #include <crypto/internal/skcipher.h>
> #include <crypto/scatterwalk.h>
> #include <linux/err.h>
> @@ -23,11 +24,11 @@
>
> struct ccm_instance_ctx {
> struct crypto_skcipher_spawn ctr;
> - struct crypto_spawn cipher;
> + struct crypto_ahash_spawn mac;
> };
>
> struct crypto_ccm_ctx {
> - struct crypto_cipher *cipher;
> + struct crypto_ahash *mac;
> struct crypto_skcipher *ctr;
> };
>
> @@ -44,15 +45,22 @@ struct crypto_rfc4309_req_ctx {
>
> struct crypto_ccm_req_priv_ctx {
> u8 odata[16];
> - u8 idata[16];
> u8 auth_tag[16];
> - u32 ilen;
> u32 flags;
> struct scatterlist src[3];
> struct scatterlist dst[3];
> struct skcipher_request skreq;
> };
>
> +struct cbcmac_tfm_ctx {
> + struct crypto_cipher *child;
> +};
> +
> +struct cbcmac_desc_ctx {
> + unsigned int len;
> + u8 dg[];
> +};
> +
> static inline struct crypto_ccm_req_priv_ctx *crypto_ccm_reqctx(
> struct aead_request *req)
> {
> @@ -84,7 +92,7 @@ static int crypto_ccm_setkey(struct crypto_aead *aead, const u8 *key,
> {
> struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
> struct crypto_skcipher *ctr = ctx->ctr;
> - struct crypto_cipher *tfm = ctx->cipher;
> + struct crypto_ahash *mac = ctx->mac;
> int err = 0;
>
> crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
> @@ -96,11 +104,11 @@ static int crypto_ccm_setkey(struct crypto_aead *aead, const u8 *key,
> if (err)
> goto out;
>
> - crypto_cipher_clear_flags(tfm, CRYPTO_TFM_REQ_MASK);
> - crypto_cipher_set_flags(tfm, crypto_aead_get_flags(aead) &
> + crypto_ahash_clear_flags(mac, CRYPTO_TFM_REQ_MASK);
> + crypto_ahash_set_flags(mac, crypto_aead_get_flags(aead) &
> CRYPTO_TFM_REQ_MASK);
> - err = crypto_cipher_setkey(tfm, key, keylen);
> - crypto_aead_set_flags(aead, crypto_cipher_get_flags(tfm) &
> + err = crypto_ahash_setkey(mac, key, keylen);
> + crypto_aead_set_flags(aead, crypto_ahash_get_flags(mac) &
> CRYPTO_TFM_RES_MASK);
>
> out:
> @@ -167,119 +175,61 @@ static int format_adata(u8 *adata, unsigned int a)
> return len;
> }
>
> -static void compute_mac(struct crypto_cipher *tfm, u8 *data, int n,
> - struct crypto_ccm_req_priv_ctx *pctx)
> -{
> - unsigned int bs = 16;
> - u8 *odata = pctx->odata;
> - u8 *idata = pctx->idata;
> - int datalen, getlen;
> -
> - datalen = n;
> -
> - /* first time in here, block may be partially filled. */
> - getlen = bs - pctx->ilen;
> - if (datalen >= getlen) {
> - memcpy(idata + pctx->ilen, data, getlen);
> - crypto_xor(odata, idata, bs);
> - crypto_cipher_encrypt_one(tfm, odata, odata);
> - datalen -= getlen;
> - data += getlen;
> - pctx->ilen = 0;
> - }
> -
> - /* now encrypt rest of data */
> - while (datalen >= bs) {
> - crypto_xor(odata, data, bs);
> - crypto_cipher_encrypt_one(tfm, odata, odata);
> -
> - datalen -= bs;
> - data += bs;
> - }
> -
> - /* check and see if there's leftover data that wasn't
> - * enough to fill a block.
> - */
> - if (datalen) {
> - memcpy(idata + pctx->ilen, data, datalen);
> - pctx->ilen += datalen;
> - }
> -}
> -
> -static void get_data_to_compute(struct crypto_cipher *tfm,
> - struct crypto_ccm_req_priv_ctx *pctx,
> - struct scatterlist *sg, unsigned int len)
> -{
> - struct scatter_walk walk;
> - u8 *data_src;
> - int n;
> -
> - scatterwalk_start(&walk, sg);
> -
> - while (len) {
> - n = scatterwalk_clamp(&walk, len);
> - if (!n) {
> - scatterwalk_start(&walk, sg_next(walk.sg));
> - n = scatterwalk_clamp(&walk, len);
> - }
> - data_src = scatterwalk_map(&walk);
> -
> - compute_mac(tfm, data_src, n, pctx);
> - len -= n;
> -
> - scatterwalk_unmap(data_src);
> - scatterwalk_advance(&walk, n);
> - scatterwalk_done(&walk, 0, len);
> - if (len)
> - crypto_yield(pctx->flags);
> - }
> -
> - /* any leftover needs padding and then encrypted */
> - if (pctx->ilen) {
> - int padlen;
> - u8 *odata = pctx->odata;
> - u8 *idata = pctx->idata;
> -
> - padlen = 16 - pctx->ilen;
> - memset(idata + pctx->ilen, 0, padlen);
> - crypto_xor(odata, idata, 16);
> - crypto_cipher_encrypt_one(tfm, odata, odata);
> - pctx->ilen = 0;
> - }
> -}
> -
> static int crypto_ccm_auth(struct aead_request *req, struct scatterlist *plain,
> unsigned int cryptlen)
> {
> + struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
> struct crypto_aead *aead = crypto_aead_reqtfm(req);
> struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
> - struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
> - struct crypto_cipher *cipher = ctx->cipher;
> + AHASH_REQUEST_ON_STACK(ahreq, ctx->mac);
> unsigned int assoclen = req->assoclen;
> - u8 *odata = pctx->odata;
> - u8 *idata = pctx->idata;
> - int err;
> + struct scatterlist sg[3];
> + u8 odata[16];
> + u8 idata[6];
> + int ilen, err;
>
> /* format control data for input */
> err = format_input(odata, req, cryptlen);
> if (err)
> goto out;
>
> - /* encrypt first block to use as start in computing mac */
> - crypto_cipher_encrypt_one(cipher, odata, odata);
> + sg_init_table(sg, 3);
> + sg_set_buf(&sg[0], odata, 16);
>
> /* format associated data and compute into mac */
> if (assoclen) {
> - pctx->ilen = format_adata(idata, assoclen);
> - get_data_to_compute(cipher, pctx, req->src, req->assoclen);
> + ilen = format_adata(idata, assoclen);
> + sg_set_buf(&sg[1], idata, ilen);
> + sg_chain(sg, 3, req->src);
> } else {
> - pctx->ilen = 0;
> + ilen = 0;
> + sg_chain(sg, 2, req->src);
> }
>
> - /* compute plaintext into mac */
> - if (cryptlen)
> - get_data_to_compute(cipher, pctx, plain, cryptlen);
> + ahash_request_set_tfm(ahreq, ctx->mac);
> + ahash_request_set_callback(ahreq, pctx->flags, NULL, NULL);
> + ahash_request_set_crypt(ahreq, sg, NULL, assoclen + ilen + 16);
> + err = crypto_ahash_init(ahreq);
> + if (err)
> + goto out;
> + err = crypto_ahash_update(ahreq);
> + if (err)
> + goto out;
>
> + /* we need to pad the MAC input to a round multiple of the block size */
> + ilen = 16 - (assoclen + ilen) % 16;
> + if (ilen < 16) {
> + memset(idata, 0, ilen);
> + sg_init_table(sg, 2);
> + sg_set_buf(&sg[0], idata, ilen);
> + if (plain)
> + sg_chain(sg, 2, plain);
> + plain = sg;
> + cryptlen += ilen;
> + }
> +
> + ahash_request_set_crypt(ahreq, plain, pctx->odata, cryptlen);
> + err = crypto_ahash_finup(ahreq);
> out:
> return err;
> }
> @@ -453,21 +403,21 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
> struct aead_instance *inst = aead_alg_instance(tfm);
> struct ccm_instance_ctx *ictx = aead_instance_ctx(inst);
> struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm);
> - struct crypto_cipher *cipher;
> + struct crypto_ahash *mac;
> struct crypto_skcipher *ctr;
> unsigned long align;
> int err;
>
> - cipher = crypto_spawn_cipher(&ictx->cipher);
> - if (IS_ERR(cipher))
> - return PTR_ERR(cipher);
> + mac = crypto_spawn_ahash(&ictx->mac);
> + if (IS_ERR(mac))
> + return PTR_ERR(mac);
>
> ctr = crypto_spawn_skcipher(&ictx->ctr);
> err = PTR_ERR(ctr);
> if (IS_ERR(ctr))
> - goto err_free_cipher;
> + goto err_free_mac;
>
> - ctx->cipher = cipher;
> + ctx->mac = mac;
> ctx->ctr = ctr;
>
> align = crypto_aead_alignmask(tfm);
> @@ -479,8 +429,8 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
>
> return 0;
>
> -err_free_cipher:
> - crypto_free_cipher(cipher);
> +err_free_mac:
> + crypto_free_ahash(mac);
> return err;
> }
>
> @@ -488,7 +438,7 @@ static void crypto_ccm_exit_tfm(struct crypto_aead *tfm)
> {
> struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm);
>
> - crypto_free_cipher(ctx->cipher);
> + crypto_free_ahash(ctx->mac);
> crypto_free_skcipher(ctx->ctr);
> }
>
> @@ -496,7 +446,7 @@ static void crypto_ccm_free(struct aead_instance *inst)
> {
> struct ccm_instance_ctx *ctx = aead_instance_ctx(inst);
>
> - crypto_drop_spawn(&ctx->cipher);
> + crypto_drop_ahash(&ctx->mac);
> crypto_drop_skcipher(&ctx->ctr);
> kfree(inst);
> }
> @@ -505,12 +455,13 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> struct rtattr **tb,
> const char *full_name,
> const char *ctr_name,
> - const char *cipher_name)
> + const char *mac_name)
> {
> struct crypto_attr_type *algt;
> struct aead_instance *inst;
> struct skcipher_alg *ctr;
> - struct crypto_alg *cipher;
> + struct crypto_alg *mac_alg;
> + struct hash_alg_common *mac;
> struct ccm_instance_ctx *ictx;
> int err;
>
> @@ -521,25 +472,26 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
> return -EINVAL;
>
> - cipher = crypto_alg_mod_lookup(cipher_name, CRYPTO_ALG_TYPE_CIPHER,
> - CRYPTO_ALG_TYPE_MASK);
> - if (IS_ERR(cipher))
> - return PTR_ERR(cipher);
> + mac_alg = crypto_find_alg(mac_name, &crypto_ahash_type,
> + CRYPTO_ALG_TYPE_HASH,
> + CRYPTO_ALG_TYPE_AHASH_MASK |
> + CRYPTO_ALG_ASYNC);
> + if (IS_ERR(mac_alg))
> + return PTR_ERR(mac_alg);
>
> + mac = __crypto_hash_alg_common(mac_alg);
> err = -EINVAL;
> - if (cipher->cra_blocksize != 16)
> - goto out_put_cipher;
> + if (mac->digestsize != 16)
> + goto out_put_mac;
>
> inst = kzalloc(sizeof(*inst) + sizeof(*ictx), GFP_KERNEL);
> err = -ENOMEM;
> if (!inst)
> - goto out_put_cipher;
> + goto out_put_mac;
>
> ictx = aead_instance_ctx(inst);
> -
> - err = crypto_init_spawn(&ictx->cipher, cipher,
> - aead_crypto_instance(inst),
> - CRYPTO_ALG_TYPE_MASK);
> + err = crypto_init_ahash_spawn(&ictx->mac, mac,
> + aead_crypto_instance(inst));
> if (err)
> goto err_free_inst;
>
> @@ -548,7 +500,7 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> crypto_requires_sync(algt->type,
> algt->mask));
> if (err)
> - goto err_drop_cipher;
> + goto err_drop_mac;
>
> ctr = crypto_spawn_skcipher_alg(&ictx->ctr);
>
> @@ -564,16 +516,16 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> err = -ENAMETOOLONG;
> if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
> "ccm_base(%s,%s)", ctr->base.cra_driver_name,
> - cipher->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
> + mac->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
> goto err_drop_ctr;
>
> memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
>
> inst->alg.base.cra_flags = ctr->base.cra_flags & CRYPTO_ALG_ASYNC;
> - inst->alg.base.cra_priority = (cipher->cra_priority +
> + inst->alg.base.cra_priority = (mac->base.cra_priority +
> ctr->base.cra_priority) / 2;
> inst->alg.base.cra_blocksize = 1;
> - inst->alg.base.cra_alignmask = cipher->cra_alignmask |
> + inst->alg.base.cra_alignmask = mac->base.cra_alignmask |
> ctr->base.cra_alignmask |
> (__alignof__(u32) - 1);
> inst->alg.ivsize = 16;
> @@ -593,23 +545,24 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> if (err)
> goto err_drop_ctr;
>
> -out_put_cipher:
> - crypto_mod_put(cipher);
> +out_put_mac:
> + crypto_mod_put(mac_alg);
> return err;
>
> err_drop_ctr:
> crypto_drop_skcipher(&ictx->ctr);
> -err_drop_cipher:
> - crypto_drop_spawn(&ictx->cipher);
> +err_drop_mac:
> + crypto_drop_ahash(&ictx->mac);
> err_free_inst:
> kfree(inst);
> - goto out_put_cipher;
> + goto out_put_mac;
> }
>
> static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb)
> {
> const char *cipher_name;
> char ctr_name[CRYPTO_MAX_ALG_NAME];
> + char mac_name[CRYPTO_MAX_ALG_NAME];
> char full_name[CRYPTO_MAX_ALG_NAME];
>
> cipher_name = crypto_attr_alg_name(tb[1]);
> @@ -620,12 +573,16 @@ static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb)
> cipher_name) >= CRYPTO_MAX_ALG_NAME)
> return -ENAMETOOLONG;
>
> + if (snprintf(mac_name, CRYPTO_MAX_ALG_NAME, "cbcmac(%s)",
> + cipher_name) >= CRYPTO_MAX_ALG_NAME)
> + return -ENAMETOOLONG;
> +
> if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm(%s)", cipher_name) >=
> CRYPTO_MAX_ALG_NAME)
> return -ENAMETOOLONG;
>
> return crypto_ccm_create_common(tmpl, tb, full_name, ctr_name,
> - cipher_name);
> + mac_name);
> }
>
> static struct crypto_template crypto_ccm_tmpl = {
> @@ -899,14 +856,156 @@ static struct crypto_template crypto_rfc4309_tmpl = {
> .module = THIS_MODULE,
> };
>
> +static int crypto_cbcmac_digest_setkey(struct crypto_shash *parent,
> + const u8 *inkey, unsigned int keylen)
> +{
> + struct cbcmac_tfm_ctx *ctx = crypto_shash_ctx(parent);
> +
> + return crypto_cipher_setkey(ctx->child, inkey, keylen);
> +}
> +
> +static int crypto_cbcmac_digest_init(struct shash_desc *pdesc)
> +{
> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
> + int bs = crypto_shash_digestsize(pdesc->tfm);
> +
> + ctx->len = 0;
> + memset(ctx->dg, 0, bs);
> +
> + return 0;
> +}
> +
> +static int crypto_cbcmac_digest_update(struct shash_desc *pdesc, const u8 *p,
> + unsigned int len)
> +{
> + struct crypto_shash *parent = pdesc->tfm;
> + struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent);
> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
> + struct crypto_cipher *tfm = tctx->child;
> + int bs = crypto_shash_digestsize(parent);
> +
> + while (len--) {
> + ctx->dg[ctx->len++] ^= *p++;
> +
> + if (ctx->len == bs) {
> + crypto_cipher_encrypt_one(tfm, ctx->dg, ctx->dg);
> + ctx->len = 0;
> + }
> + }
> +
> + return 0;
> +}
> +
> +static int crypto_cbcmac_digest_final(struct shash_desc *pdesc, u8 *out)
> +{
> + struct crypto_shash *parent = pdesc->tfm;
> + struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent);
> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
> + struct crypto_cipher *tfm = tctx->child;
> + int bs = crypto_shash_digestsize(parent);
> +
> + if (ctx->len)
> + crypto_cipher_encrypt_one(tfm, out, ctx->dg);
> + else
> + memcpy(out, ctx->dg, bs);
> +
> + return 0;
> +}
> +
> +static int cbcmac_init_tfm(struct crypto_tfm *tfm)
> +{
> + struct crypto_cipher *cipher;
> + struct crypto_instance *inst = (void *)tfm->__crt_alg;
> + struct crypto_spawn *spawn = crypto_instance_ctx(inst);
> + struct cbcmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
> +
> + cipher = crypto_spawn_cipher(spawn);
> + if (IS_ERR(cipher))
> + return PTR_ERR(cipher);
> +
> + ctx->child = cipher;
> +
> + return 0;
> +};
> +
> +static void cbcmac_exit_tfm(struct crypto_tfm *tfm)
> +{
> + struct cbcmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
> + crypto_free_cipher(ctx->child);
> +}
> +
> +static int cbcmac_create(struct crypto_template *tmpl, struct rtattr **tb)
> +{
> + struct shash_instance *inst;
> + struct crypto_alg *alg;
> + int err;
> +
> + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
> + if (err)
> + return err;
> +
> + alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
> + CRYPTO_ALG_TYPE_MASK);
> + if (IS_ERR(alg))
> + return PTR_ERR(alg);
> +
> + inst = shash_alloc_instance("cbcmac", alg);
> + err = PTR_ERR(inst);
> + if (IS_ERR(inst))
> + goto out_put_alg;
> +
> + err = crypto_init_spawn(shash_instance_ctx(inst), alg,
> + shash_crypto_instance(inst),
> + CRYPTO_ALG_TYPE_MASK);
> + if (err)
> + goto out_free_inst;
> +
> + inst->alg.base.cra_priority = alg->cra_priority;
> + inst->alg.base.cra_blocksize = 1;
> +
> + inst->alg.digestsize = alg->cra_blocksize;
> + inst->alg.descsize = sizeof(struct cbcmac_desc_ctx) +
> + alg->cra_blocksize;
> +
> + inst->alg.base.cra_ctxsize = sizeof(struct cbcmac_tfm_ctx);
> + inst->alg.base.cra_init = cbcmac_init_tfm;
> + inst->alg.base.cra_exit = cbcmac_exit_tfm;
> +
> + inst->alg.init = crypto_cbcmac_digest_init;
> + inst->alg.update = crypto_cbcmac_digest_update;
> + inst->alg.final = crypto_cbcmac_digest_final;
> + inst->alg.setkey = crypto_cbcmac_digest_setkey;
> +
> + err = shash_register_instance(tmpl, inst);
> +
> +out_free_inst:
> + if (err)
> + shash_free_instance(shash_crypto_instance(inst));
> +
> +out_put_alg:
> + crypto_mod_put(alg);
> + return err;
> +}
> +
> +static struct crypto_template crypto_cbcmac_tmpl = {
> + .name = "cbcmac",
> + .create = cbcmac_create,
> + .free = shash_free_instance,
> + .module = THIS_MODULE,
> +};
> +
> static int __init crypto_ccm_module_init(void)
> {
> int err;
>
> - err = crypto_register_template(&crypto_ccm_base_tmpl);
> + err = crypto_register_template(&crypto_cbcmac_tmpl);
> if (err)
> goto out;
>
> + err = crypto_register_template(&crypto_ccm_base_tmpl);
> + if (err)
> + goto out_undo_cbcmac;
> +
> err = crypto_register_template(&crypto_ccm_tmpl);
> if (err)
> goto out_undo_base;
> @@ -922,6 +1021,8 @@ static int __init crypto_ccm_module_init(void)
> crypto_unregister_template(&crypto_ccm_tmpl);
> out_undo_base:
> crypto_unregister_template(&crypto_ccm_base_tmpl);
> +out_undo_cbcmac:
> + crypto_register_template(&crypto_cbcmac_tmpl);
> goto out;
> }
>
> @@ -930,6 +1031,7 @@ static void __exit crypto_ccm_module_exit(void)
> crypto_unregister_template(&crypto_rfc4309_tmpl);
> crypto_unregister_template(&crypto_ccm_tmpl);
> crypto_unregister_template(&crypto_ccm_base_tmpl);
> + crypto_unregister_template(&crypto_cbcmac_tmpl);
> }
>
> module_init(crypto_ccm_module_init);
> --
> 2.7.4
>
^ permalink raw reply
* Re: [PATCH v5 2/3] drivers: crypto: Add the Virtual Function driver for CPT
From: Sasha Levin @ 2017-02-02 18:54 UTC (permalink / raw)
To: George Cherian
Cc: Herbert Xu, davem, david.daney, clabbe.montjoie, smueller,
linux-kernel@vger.kernel.org List, linux-crypto, alexander.levin
In-Reply-To: <1485779444-4332-3-git-send-email-george.cherian@cavium.com>
On Mon, Jan 30, 2017 at 7:30 AM, George Cherian
<george.cherian@cavium.com> wrote:
> diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c
> new file mode 100644
> index 0000000..4cf466d
> --- /dev/null
> +++ b/drivers/crypto/cavium/cpt/cptvf_main.c
> @@ -0,0 +1,948 @@
> +/*
> + * Copyright (C) 2016 Cavium, Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of version 2 of the GNU General Public License
> + * as published by the Free Software Foundation.
> + */
> +
> +#include <linux/interrupt.h>
> +#include <linux/module.h>
> +
> +#include "cptvf.h"
> +
> +#define DRV_NAME "thunder-cptvf"
> +#define DRV_VERSION "1.0"
> +
> +struct cptvf_wqe {
> + struct tasklet_struct twork;
> + void *cptvf;
> + u32 qno;
> +};
> +
> +struct cptvf_wqe_info {
> + struct cptvf_wqe vq_wqe[CPT_NUM_QS_PER_VF];
> +};
> +
> +static void vq_work_handler(unsigned long data)
> +{
> + struct cptvf_wqe_info *cwqe_info = (struct cptvf_wqe_info *)data;
> + struct cptvf_wqe *cwqe = &cwqe_info->vq_wqe[0];
> +
> + vq_post_process(cwqe->cptvf, cwqe->qno);
> +}
> +
> +static int init_worker_threads(struct cpt_vf *cptvf)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + struct cptvf_wqe_info *cwqe_info;
> + int i;
> +
> + cwqe_info = kzalloc(sizeof(*cwqe_info), GFP_KERNEL);
> + if (!cwqe_info)
> + return -ENOMEM;
> +
> + if (cptvf->nr_queues) {
> + dev_info(&pdev->dev, "Creating VQ worker threads (%d)\n",
> + cptvf->nr_queues);
> + }
> +
> + for (i = 0; i < cptvf->nr_queues; i++) {
> + tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler,
> + (u64)cwqe_info);
> + cwqe_info->vq_wqe[i].qno = i;
> + cwqe_info->vq_wqe[i].cptvf = cptvf;
> + }
> +
> + cptvf->wqe_info = cwqe_info;
> +
> + return 0;
> +}
> +
> +static void cleanup_worker_threads(struct cpt_vf *cptvf)
> +{
> + struct cptvf_wqe_info *cwqe_info;
> + struct pci_dev *pdev = cptvf->pdev;
> + int i;
> +
> + cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
> + if (!cwqe_info)
> + return;
> +
> + if (cptvf->nr_queues) {
> + dev_info(&pdev->dev, "Cleaning VQ worker threads (%u)\n",
> + cptvf->nr_queues);
> + }
> +
> + for (i = 0; i < cptvf->nr_queues; i++)
> + tasklet_kill(&cwqe_info->vq_wqe[i].twork);
> +
> + kzfree(cwqe_info);
> + cptvf->wqe_info = NULL;
> +}
> +
> +static void free_pending_queues(struct pending_qinfo *pqinfo)
> +{
> + int i;
> + struct pending_queue *queue;
> +
> + for_each_pending_queue(pqinfo, queue, i) {
> + if (!queue->head)
> + continue;
> +
> + /* free single queue */
> + kzfree((queue->head));
> +
> + queue->front = 0;
> + queue->rear = 0;
> +
> + return;
> + }
> +
> + pqinfo->qlen = 0;
> + pqinfo->nr_queues = 0;
> +}
> +
> +static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen,
> + u32 nr_queues)
> +{
> + u32 i;
> + size_t size;
> + int ret;
> + struct pending_queue *queue = NULL;
> +
> + pqinfo->nr_queues = nr_queues;
> + pqinfo->qlen = qlen;
> +
> + size = (qlen * sizeof(struct pending_entry));
> +
> + for_each_pending_queue(pqinfo, queue, i) {
> + queue->head = kzalloc((size), GFP_KERNEL);
> + if (!queue->head) {
> + ret = -ENOMEM;
> + goto pending_qfail;
> + }
> +
> + queue->front = 0;
> + queue->rear = 0;
> + atomic64_set((&queue->pending_count), (0));
> +
> + /* init queue spin lock */
> + spin_lock_init(&queue->lock);
> + }
> +
> + return 0;
> +
> +pending_qfail:
> + free_pending_queues(pqinfo);
> +
> + return ret;
> +}
> +
> +static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + int ret;
> +
> + if (!nr_queues)
> + return 0;
> +
> + ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues);
> + if (ret) {
> + dev_err(&pdev->dev, "failed to setup pending queues (%u)\n",
> + nr_queues);
> + return ret;
> + }
> +
> + return 0;
> +}
> +
> +static void cleanup_pending_queues(struct cpt_vf *cptvf)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> +
> + if (!cptvf->nr_queues)
> + return;
> +
> + dev_info(&pdev->dev, "Cleaning VQ pending queue (%u)\n",
> + cptvf->nr_queues);
> + free_pending_queues(&cptvf->pqinfo);
> +}
> +
> +static void free_command_queues(struct cpt_vf *cptvf,
> + struct command_qinfo *cqinfo)
> +{
> + int i, j;
> + struct command_queue *queue = NULL;
> + struct command_chunk *chunk = NULL, *next = NULL;
> + struct pci_dev *pdev = cptvf->pdev;
> + struct hlist_node *node;
> +
> + /* clean up for each queue */
> + for (i = 0; i < cptvf->nr_queues; i++) {
> + queue = &cqinfo->queue[i];
> + if (hlist_empty(&cqinfo->queue[i].chead))
> + continue;
> +
> + hlist_for_each(node, &cqinfo->queue[i].chead) {
> + chunk = hlist_entry(node, struct command_chunk,
> + nextchunk);
> + break;
> + }
What exactly is the purpose of that loop?
> + for (j = 0; j < queue->nchunks; j++) {
> + if (j < queue->nchunks) {
We already know that "j < queue->nchunks" at this point...
> + node = node->next;
> + next = hlist_entry(node, struct command_chunk,
> + nextchunk);
> + }
> +
> + dma_free_coherent(&pdev->dev, chunk->size,
> + chunk->head,
> + chunk->dma_addr);
> + chunk->head = NULL;
> + chunk->dma_addr = 0;
> + hlist_del(&chunk->nextchunk);
> + kzfree(chunk);
> + chunk = next;
> + }
> + queue->nchunks = 0;
> + queue->idx = 0;
> + }
This whole function looks like an attempt to open code
hlist_for_each_entry_safe(), why didn't you just use that?
> +
> + /* common cleanup */
> + cqinfo->cmd_size = 0;
> +}
> +
> +static int alloc_command_queues(struct cpt_vf *cptvf,
> + struct command_qinfo *cqinfo, size_t cmd_size,
> + u32 qlen)
> +{
> + int i;
> + size_t q_size;
> + struct command_queue *queue = NULL;
> + struct pci_dev *pdev = cptvf->pdev;
> +
> + /* common init */
> + cqinfo->cmd_size = cmd_size;
> + /* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */
> + cptvf->qsize = min(qlen, cqinfo->qchunksize) *
> + CPT_NEXT_CHUNK_PTR_SIZE + 1;
> + /* Qsize in bytes to create space for alignment */
> + q_size = qlen * cqinfo->cmd_size;
> +
> + /* per queue initialization */
> + for (i = 0; i < cptvf->nr_queues; i++) {
> + size_t c_size = 0;
> + size_t rem_q_size = q_size;
> + struct command_chunk *curr = NULL, *first = NULL, *last = NULL;
> + u32 qcsize_bytes = cqinfo->qchunksize * cqinfo->cmd_size;
> +
> + queue = &cqinfo->queue[i];
> + INIT_HLIST_HEAD(&cqinfo->queue[i].chead);
> + do {
> + curr = kzalloc(sizeof(*curr), GFP_KERNEL);
> + if (!curr)
> + goto cmd_qfail;
> +
> + c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes :
> + rem_q_size;
> + curr->head = (u8 *)dma_zalloc_coherent(&pdev->dev,
> + c_size + CPT_NEXT_CHUNK_PTR_SIZE,
> + &curr->dma_addr, GFP_KERNEL);
> + if (!curr->head) {
> + dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n",
> + i, queue->nchunks);
> + goto cmd_qfail;
> + }
> +
> + curr->size = c_size;
> + if (queue->nchunks == 0) {
> + hlist_add_head(&curr->nextchunk,
> + &cqinfo->queue[i].chead);
> + first = curr;
> + } else {
> + hlist_add_behind(&curr->nextchunk,
> + &last->nextchunk);
> + }
> +
> + queue->nchunks++;
> + rem_q_size -= c_size;
> + if (last)
> + *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
> +
> + last = curr;
> + } while (rem_q_size);
> +
> + /* Make the queue circular */
> + /* Tie back last chunk entry to head */
> + curr = first;
> + *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
> + last->nextchunk.next = &curr->nextchunk;
You shouldn't access the hlist struct members directly, use helper
functions here.
> + queue->qhead = curr;
> + spin_lock_init(&queue->lock);
> + }
> + return 0;
> +
> +cmd_qfail:
> + free_command_queues(cptvf, cqinfo);
> + return -ENOMEM;
> +}
> +
> +static int init_command_queues(struct cpt_vf *cptvf, u32 qlen)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + int ret;
> +
> + /* setup AE command queues */
> + ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE,
> + qlen);
> + if (ret) {
> + dev_err(&pdev->dev, "failed to allocate AE command queues (%u)\n",
> + cptvf->nr_queues);
> + return ret;
> + }
> +
> + return ret;
> +}
> +
> +static void cleanup_command_queues(struct cpt_vf *cptvf)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> +
> + if (!cptvf->nr_queues)
> + return;
> +
> + dev_info(&pdev->dev, "Cleaning VQ command queue (%u)\n",
> + cptvf->nr_queues);
> + free_command_queues(cptvf, &cptvf->cqinfo);
> +}
> +
> +static void cptvf_sw_cleanup(struct cpt_vf *cptvf)
> +{
> + cleanup_worker_threads(cptvf);
> + cleanup_pending_queues(cptvf);
> + cleanup_command_queues(cptvf);
> +}
> +
> +static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + int ret = 0;
> + u32 max_dev_queues = 0;
> +
> + max_dev_queues = CPT_NUM_QS_PER_VF;
> + /* possible cpus */
> + nr_queues = min_t(u32, nr_queues, max_dev_queues);
> + cptvf->nr_queues = nr_queues;
> +
> + ret = init_command_queues(cptvf, qlen);
> + if (ret) {
> + dev_err(&pdev->dev, "Failed to setup command queues (%u)\n",
> + nr_queues);
> + return ret;
> + }
> +
> + ret = init_pending_queues(cptvf, qlen, nr_queues);
> + if (ret) {
> + dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n",
> + nr_queues);
> + goto setup_pqfail;
> + }
> +
> + /* Create worker threads for BH processing */
> + ret = init_worker_threads(cptvf);
> + if (ret) {
> + dev_err(&pdev->dev, "Failed to setup worker threads\n");
> + goto init_work_fail;
> + }
> +
> + return 0;
> +
> +init_work_fail:
> + cleanup_worker_threads(cptvf);
> + cleanup_pending_queues(cptvf);
> +
> +setup_pqfail:
> + cleanup_command_queues(cptvf);
> +
> + return ret;
> +}
> +
> +static void cptvf_disable_msix(struct cpt_vf *cptvf)
> +{
> + if (cptvf->msix_enabled) {
> + pci_disable_msix(cptvf->pdev);
> + cptvf->msix_enabled = 0;
> + }
> +}
> +
> +static int cptvf_enable_msix(struct cpt_vf *cptvf)
> +{
> + int i, ret;
> +
> + for (i = 0; i < CPT_VF_MSIX_VECTORS; i++)
> + cptvf->msix_entries[i].entry = i;
> +
> + ret = pci_enable_msix(cptvf->pdev, cptvf->msix_entries,
> + CPT_VF_MSIX_VECTORS);
> + if (ret) {
> + dev_err(&cptvf->pdev->dev, "Request for #%d msix vectors failed\n",
> + CPT_VF_MSIX_VECTORS);
> + return ret;
> + }
> +
> + cptvf->msix_enabled = 1;
> + /* Mark MSIX enabled */
> + cptvf->flags |= CPT_FLAG_MSIX_ENABLED;
> +
> + return 0;
> +}
> +
> +static void cptvf_free_all_interrupts(struct cpt_vf *cptvf)
> +{
> + int irq;
> +
> + for (irq = 0; irq < CPT_VF_MSIX_VECTORS; irq++) {
> + if (cptvf->irq_allocated[irq])
> + irq_set_affinity_hint(cptvf->msix_entries[irq].vector,
> + NULL);
> + free_cpumask_var(cptvf->affinity_mask[irq]);
> + free_irq(cptvf->msix_entries[irq].vector, cptvf);
> + cptvf->irq_allocated[irq] = false;
> + }
> +}
> +
> +static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val)
> +{
> + union cptx_vqx_ctl vqx_ctl;
> +
> + vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0));
> + vqx_ctl.s.ena = val;
> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u);
> +}
> +
> +void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val)
> +{
> + union cptx_vqx_doorbell vqx_dbell;
> +
> + vqx_dbell.u = cpt_read_csr64(cptvf->reg_base,
> + CPTX_VQX_DOORBELL(0, 0));
> + vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */
> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0),
> + vqx_dbell.u);
> +}
> +
> +static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val)
> +{
> + union cptx_vqx_inprog vqx_inprg;
> +
> + vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0));
> + vqx_inprg.s.inflight = val;
> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u);
> +}
> +
> +static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val)
> +{
> + union cptx_vqx_done_wait vqx_dwait;
> +
> + vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
> + CPTX_VQX_DONE_WAIT(0, 0));
> + vqx_dwait.s.num_wait = val;
> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
> + vqx_dwait.u);
> +}
> +
> +static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time)
> +{
> + union cptx_vqx_done_wait vqx_dwait;
> +
> + vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
> + CPTX_VQX_DONE_WAIT(0, 0));
> + vqx_dwait.s.time_wait = time;
> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
> + vqx_dwait.u);
> +}
> +
> +static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf)
> +{
> + union cptx_vqx_misc_ena_w1s vqx_misc_ena;
> +
> + vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
> + CPTX_VQX_MISC_ENA_W1S(0, 0));
> + /* Set mbox(0) interupts for the requested vf */
> + vqx_misc_ena.s.swerr = 1;
> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
> + vqx_misc_ena.u);
> +}
> +
> +static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf)
> +{
> + union cptx_vqx_misc_ena_w1s vqx_misc_ena;
> +
> + vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
> + CPTX_VQX_MISC_ENA_W1S(0, 0));
> + /* Set mbox(0) interupts for the requested vf */
> + vqx_misc_ena.s.mbox = 1;
> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
> + vqx_misc_ena.u);
> +}
> +
> +static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf)
> +{
> + union cptx_vqx_done_ena_w1s vqx_done_ena;
> +
> + vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base,
> + CPTX_VQX_DONE_ENA_W1S(0, 0));
> + /* Set DONE interrupt for the requested vf */
> + vqx_done_ena.s.done = 1;
> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0),
> + vqx_done_ena.u);
> +}
> +
> +static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf)
> +{
> + union cptx_vqx_misc_int vqx_misc_int;
> +
> + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
> + CPTX_VQX_MISC_INT(0, 0));
> + /* W1C for the VF */
> + vqx_misc_int.s.dovf = 1;
> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
> + vqx_misc_int.u);
> +}
> +
> +static void cptvf_clear_irde_intr(struct cpt_vf *cptvf)
> +{
> + union cptx_vqx_misc_int vqx_misc_int;
> +
> + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
> + CPTX_VQX_MISC_INT(0, 0));
> + /* W1C for the VF */
> + vqx_misc_int.s.irde = 1;
> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
> + vqx_misc_int.u);
> +}
> +
> +static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf)
> +{
> + union cptx_vqx_misc_int vqx_misc_int;
> +
> + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
> + CPTX_VQX_MISC_INT(0, 0));
> + /* W1C for the VF */
> + vqx_misc_int.s.nwrp = 1;
> + cpt_write_csr64(cptvf->reg_base,
> + CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
> +}
> +
> +static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf)
> +{
> + union cptx_vqx_misc_int vqx_misc_int;
> +
> + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
> + CPTX_VQX_MISC_INT(0, 0));
> + /* W1C for the VF */
> + vqx_misc_int.s.mbox = 1;
> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
> + vqx_misc_int.u);
> +}
> +
> +static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf)
> +{
> + union cptx_vqx_misc_int vqx_misc_int;
> +
> + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
> + CPTX_VQX_MISC_INT(0, 0));
> + /* W1C for the VF */
> + vqx_misc_int.s.swerr = 1;
> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
> + vqx_misc_int.u);
> +}
> +
> +static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf)
> +{
> + return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0));
> +}
> +
> +static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq)
> +{
> + struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
> + struct pci_dev *pdev = cptvf->pdev;
> + u64 intr;
> +
> + intr = cptvf_read_vf_misc_intr_status(cptvf);
> + /*Check for MISC interrupt types*/
> + if (likely(intr & CPT_VF_INTR_MBOX_MASK)) {
> + dev_err(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n",
> + intr, cptvf->vfid);
> + cptvf_handle_mbox_intr(cptvf);
> + cptvf_clear_mbox_intr(cptvf);
> + } else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) {
> + cptvf_clear_dovf_intr(cptvf);
> + /*Clear doorbell count*/
> + cptvf_write_vq_doorbell(cptvf, 0);
> + dev_err(&pdev->dev, "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n",
> + intr, cptvf->vfid);
> + } else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) {
> + cptvf_clear_irde_intr(cptvf);
> + dev_err(&pdev->dev, "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n",
> + intr, cptvf->vfid);
> + } else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) {
> + cptvf_clear_nwrp_intr(cptvf);
> + dev_err(&pdev->dev, "NCB response write error interrupt 0x%llx on CPT VF %d\n",
> + intr, cptvf->vfid);
> + } else if (unlikely(intr & CPT_VF_INTR_SERR_MASK)) {
> + cptvf_clear_swerr_intr(cptvf);
> + dev_err(&pdev->dev, "Software error interrupt 0x%llx on CPT VF %d\n",
> + intr, cptvf->vfid);
> + } else {
> + dev_err(&pdev->dev, "Unhandled interrupt in CPT VF %d\n",
> + cptvf->vfid);
> + }
> +
> + return IRQ_HANDLED;
> +}
> +
> +static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf,
> + int qno)
> +{
> + struct cptvf_wqe_info *nwqe_info;
> +
> + if (unlikely(qno >= cptvf->nr_queues))
> + return NULL;
> + nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
> +
> + return &nwqe_info->vq_wqe[qno];
> +}
> +
> +static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf)
> +{
> + union cptx_vqx_done vqx_done;
> +
> + vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0));
> + return vqx_done.s.done;
> +}
> +
> +static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf,
> + u32 ackcnt)
> +{
> + union cptx_vqx_done_ack vqx_dack_cnt;
> +
> + vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base,
> + CPTX_VQX_DONE_ACK(0, 0));
> + vqx_dack_cnt.s.done_ack = ackcnt;
> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0),
> + vqx_dack_cnt.u);
> +}
> +
> +static irqreturn_t cptvf_done_intr_handler(int irq, void *cptvf_irq)
> +{
> + struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
> + struct pci_dev *pdev = cptvf->pdev;
> + /* Read the number of completions */
> + u32 intr = cptvf_read_vq_done_count(cptvf);
> +
> + if (intr) {
> + struct cptvf_wqe *wqe;
> +
> + /* Acknowledge the number of
> + * scheduled completions for processing
> + */
> + cptvf_write_vq_done_ack(cptvf, intr);
> + wqe = get_cptvf_vq_wqe(cptvf, 0);
> + if (unlikely(!wqe)) {
> + dev_err(&pdev->dev, "No work to schedule for VF (%d)",
> + cptvf->vfid);
> + return IRQ_NONE;
> + }
> + tasklet_hi_schedule(&wqe->twork);
> + }
> +
> + return IRQ_HANDLED;
> +}
> +
> +static int cptvf_register_misc_intr(struct cpt_vf *cptvf)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + int ret;
> +
> + /* Register misc interrupt handlers */
> + ret = request_irq(cptvf->msix_entries[CPT_VF_INT_VEC_E_MISC].vector,
> + cptvf_misc_intr_handler, 0, "CPT VF misc intr",
> + cptvf);
> + if (ret)
> + goto fail;
> +
> + cptvf->irq_allocated[CPT_VF_INT_VEC_E_MISC] = true;
> +
> + /* Enable mailbox interrupt */
> + cptvf_enable_mbox_interrupts(cptvf);
> + cptvf_enable_swerr_interrupts(cptvf);
> +
> + return 0;
> +
> +fail:
> + dev_err(&pdev->dev, "Request misc irq failed");
> + cptvf_free_all_interrupts(cptvf);
> + return ret;
> +}
> +
> +static int cptvf_register_done_intr(struct cpt_vf *cptvf)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + int ret;
> +
> + /* Register DONE interrupt handlers */
> + ret = request_irq(cptvf->msix_entries[CPT_VF_INT_VEC_E_DONE].vector,
> + cptvf_done_intr_handler, 0, "CPT VF done intr",
> + cptvf);
> + if (ret)
> + goto fail;
> +
> + cptvf->irq_allocated[CPT_VF_INT_VEC_E_DONE] = true;
> +
> + /* Enable mailbox interrupt */
> + cptvf_enable_done_interrupts(cptvf);
> + return 0;
> +
> +fail:
> + dev_err(&pdev->dev, "Request done irq failed\n");
> + cptvf_free_all_interrupts(cptvf);
> + return ret;
> +}
> +
> +static void cptvf_unregister_interrupts(struct cpt_vf *cptvf)
> +{
> + cptvf_free_all_interrupts(cptvf);
> + cptvf_disable_msix(cptvf);
> +}
> +
> +static void cptvf_set_irq_affinity(struct cpt_vf *cptvf)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + int vec, cpu;
> + int irqnum;
> +
> + for (vec = 0; vec < CPT_VF_MSIX_VECTORS; vec++) {
> + if (!cptvf->irq_allocated[vec])
> + continue;
> +
> + if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec],
> + GFP_KERNEL)) {
> + dev_err(&pdev->dev, "Allocation failed for affinity_mask for VF %d",
> + cptvf->vfid);
> + return;
> + }
> +
> + cpu = cptvf->vfid % num_online_cpus();
> + cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node),
> + cptvf->affinity_mask[vec]);
> + irqnum = cptvf->msix_entries[vec].vector;
> + irq_set_affinity_hint(irqnum, cptvf->affinity_mask[vec]);
> + }
> +}
> +
> +static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val)
> +{
> + union cptx_vqx_saddr vqx_saddr;
> +
> + vqx_saddr.u = val;
> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u);
> +}
> +
> +void cptvf_device_init(struct cpt_vf *cptvf)
> +{
> + u64 base_addr = 0;
> +
> + /* Disable the VQ */
> + cptvf_write_vq_ctl(cptvf, 0);
> + /* Reset the doorbell */
> + cptvf_write_vq_doorbell(cptvf, 0);
> + /* Clear inflight */
> + cptvf_write_vq_inprog(cptvf, 0);
> + /* Write VQ SADDR */
> + /* TODO: for now only one queue, so hard coded */
> + base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr);
> + cptvf_write_vq_saddr(cptvf, base_addr);
> + /* Configure timerhold / coalescence */
> + cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD);
> + cptvf_write_vq_done_numwait(cptvf, 1);
> + /* Enable the VQ */
> + cptvf_write_vq_ctl(cptvf, 1);
> + /* Flag the VF ready */
> + cptvf->flags |= CPT_FLAG_DEVICE_READY;
> +}
> +
> +static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
> +{
> + struct device *dev = &pdev->dev;
> + struct cpt_vf *cptvf;
> + int err;
> +
> + cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL);
> + if (!cptvf)
> + return -ENOMEM;
> +
> + pci_set_drvdata(pdev, cptvf);
> + cptvf->pdev = pdev;
> + err = pci_enable_device(pdev);
> + if (err) {
> + dev_err(dev, "Failed to enable PCI device\n");
> + pci_set_drvdata(pdev, NULL);
> + return err;
> + }
> +
> + err = pci_request_regions(pdev, DRV_NAME);
> + if (err) {
> + dev_err(dev, "PCI request regions failed 0x%x\n", err);
> + goto cptvf_err_disable_device;
> + }
> + /* Mark as VF driver */
> + cptvf->flags |= CPT_FLAG_VF_DRIVER;
> + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
> + if (err) {
> + dev_err(dev, "Unable to get usable DMA configuration\n");
> + goto cptvf_err_release_regions;
> + }
> +
> + err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
> + if (err) {
> + dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n");
> + goto cptvf_err_release_regions;
> + }
> +
> + /* MAP PF's configuration registers */
> + cptvf->reg_base = pcim_iomap(pdev, 0, 0);
> + if (!cptvf->reg_base) {
> + dev_err(dev, "Cannot map config register space, aborting\n");
> + err = -ENOMEM;
> + goto cptvf_err_release_regions;
> + }
> +
> + cptvf->node = dev_to_node(&pdev->dev);
> + /* Enable MSI-X */
> + err = cptvf_enable_msix(cptvf);
> + if (err) {
> + dev_err(dev, "cptvf_enable_msix() failed");
> + goto cptvf_err_release_regions;
> + }
> +
> + /* Register mailbox interrupts */
> + cptvf_register_misc_intr(cptvf);
> +
> + /* Check ready with PF */
> + /* Gets chip ID / device Id from PF if ready */
> + err = cptvf_check_pf_ready(cptvf);
> + if (err) {
> + dev_err(dev, "PF not responding to READY msg");
> + goto cptvf_err_release_regions;
> + }
> +
> + /* CPT VF software resources initialization */
> + cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE;
> + err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF);
> + if (err) {
> + dev_err(dev, "cptvf_sw_init() failed");
> + goto cptvf_err_release_regions;
> + }
> + /* Convey VQ LEN to PF */
> + err = cptvf_send_vq_size_msg(cptvf);
> + if (err) {
> + dev_err(dev, "PF not responding to QLEN msg");
> + goto cptvf_err_release_regions;
> + }
> +
> + /* CPT VF device initialization */
> + cptvf_device_init(cptvf);
> + /* Send msg to PF to assign currnet Q to required group */
> + cptvf->vfgrp = 1;
> + err = cptvf_send_vf_to_grp_msg(cptvf);
> + if (err) {
> + dev_err(dev, "PF not responding to VF_GRP msg");
> + goto cptvf_err_release_regions;
> + }
> +
> + cptvf->priority = 1;
> + err = cptvf_send_vf_priority_msg(cptvf);
> + if (err) {
> + dev_err(dev, "PF not responding to VF_PRIO msg");
> + goto cptvf_err_release_regions;
> + }
> + /* Register DONE interrupts */
> + err = cptvf_register_done_intr(cptvf);
> + if (err)
> + goto cptvf_err_release_regions;
> +
> + /* Set irq affinity masks */
> + cptvf_set_irq_affinity(cptvf);
> + /* Convey UP to PF */
> + err = cptvf_send_vf_up(cptvf);
> + if (err) {
> + dev_err(dev, "PF not responding to UP msg");
> + goto cptvf_up_fail;
> + }
> + err = cvm_crypto_init(cptvf);
> + if (err) {
> + dev_err(dev, "Algorithm register failed\n");
> + goto cptvf_up_fail;
> + }
> + return 0;
> +
> +cptvf_up_fail:
> + cptvf_unregister_interrupts(cptvf);
> +cptvf_err_release_regions:
> + pci_release_regions(pdev);
> +cptvf_err_disable_device:
> + pci_disable_device(pdev);
> + pci_set_drvdata(pdev, NULL);
> +
> + return err;
> +}
> +
> +static void cptvf_remove(struct pci_dev *pdev)
> +{
> + struct cpt_vf *cptvf = pci_get_drvdata(pdev);
> +
> + if (!cptvf)
> + dev_err(&pdev->dev, "Invalid CPT-VF device\n");
> +
> + /* Convey DOWN to PF */
> + if (cptvf_send_vf_down(cptvf)) {
> + dev_err(&pdev->dev, "PF not responding to DOWN msg");
> + } else {
> + cptvf_unregister_interrupts(cptvf);
> + cptvf_sw_cleanup(cptvf);
> + pci_set_drvdata(pdev, NULL);
> + pci_release_regions(pdev);
> + pci_disable_device(pdev);
> + cvm_crypto_exit();
> + }
> +}
> +
> +static void cptvf_shutdown(struct pci_dev *pdev)
> +{
> + cptvf_remove(pdev);
> +}
> +
> +/* Supported devices */
> +static const struct pci_device_id cptvf_id_table[] = {
> + {PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID), 0},
> + { 0, } /* end of table */
> +};
> +
> +static struct pci_driver cptvf_pci_driver = {
> + .name = DRV_NAME,
> + .id_table = cptvf_id_table,
> + .probe = cptvf_probe,
> + .remove = cptvf_remove,
> + .shutdown = cptvf_shutdown,
> +};
> +
> +module_pci_driver(cptvf_pci_driver);
> +
> +MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
> +MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver");
> +MODULE_LICENSE("GPL v2");
> +MODULE_VERSION(DRV_VERSION);
> +MODULE_DEVICE_TABLE(pci, cptvf_id_table);
> diff --git a/drivers/crypto/cavium/cpt/cptvf_mbox.c b/drivers/crypto/cavium/cpt/cptvf_mbox.c
> new file mode 100644
> index 0000000..d5ec3b8
> --- /dev/null
> +++ b/drivers/crypto/cavium/cpt/cptvf_mbox.c
> @@ -0,0 +1,211 @@
> +/*
> + * Copyright (C) 2016 Cavium, Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of version 2 of the GNU General Public License
> + * as published by the Free Software Foundation.
> + */
> +
> +#include "cptvf.h"
> +
> +static void cptvf_send_msg_to_pf(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
> +{
> + /* Writing mbox(1) causes interrupt */
> + cpt_write_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 0),
> + mbx->msg);
> + cpt_write_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 1),
> + mbx->data);
> +}
> +
> +/* ACKs PF's mailbox message
> + */
> +void cptvf_mbox_send_ack(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
> +{
> + mbx->msg = CPT_MBOX_MSG_TYPE_ACK;
> + cptvf_send_msg_to_pf(cptvf, mbx);
> +}
> +
> +/* NACKs PF's mailbox message that VF is not able to
> + * complete the action
> + */
> +void cptvf_mbox_send_nack(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
> +{
> + mbx->msg = CPT_MBOX_MSG_TYPE_NACK;
> + cptvf_send_msg_to_pf(cptvf, mbx);
> +}
> +
> +/* Interrupt handler to handle mailbox messages from VFs */
> +void cptvf_handle_mbox_intr(struct cpt_vf *cptvf)
> +{
> + struct cpt_mbox mbx = {};
> +
> + /*
> + * MBOX[0] contains msg
> + * MBOX[1] contains data
> + */
> + mbx.msg = cpt_read_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 0));
> + mbx.data = cpt_read_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 1));
> + dev_dbg(&cptvf->pdev->dev, "%s: Mailbox msg 0x%llx from PF\n",
> + __func__, mbx.msg);
> + switch (mbx.msg) {
> + case CPT_MSG_READY:
> + {
> + cptvf->pf_acked = true;
> + cptvf->vfid = mbx.data;
> + dev_dbg(&cptvf->pdev->dev, "Received VFID %d\n", cptvf->vfid);
> + break;
> + }
> + case CPT_MSG_QBIND_GRP:
> + cptvf->pf_acked = true;
> + cptvf->vftype = mbx.data;
> + dev_dbg(&cptvf->pdev->dev, "VF %d type %s group %d\n",
> + cptvf->vfid, ((mbx.data == SE_TYPES) ? "SE" : "AE"),
> + cptvf->vfgrp);
> + break;
> + case CPT_MBOX_MSG_TYPE_ACK:
> + cptvf->pf_acked = true;
> + break;
> + case CPT_MBOX_MSG_TYPE_NACK:
> + cptvf->pf_nacked = true;
> + break;
> + default:
> + dev_err(&cptvf->pdev->dev, "Invalid msg from PF, msg 0x%llx\n",
> + mbx.msg);
> + break;
> + }
> +}
> +
> +static int cptvf_send_msg_to_pf_timeout(struct cpt_vf *cptvf,
> + struct cpt_mbox *mbx)
> +{
> + int timeout = CPT_MBOX_MSG_TIMEOUT;
> + int sleep = 10;
> +
> + cptvf->pf_acked = false;
> + cptvf->pf_nacked = false;
> + cptvf_send_msg_to_pf(cptvf, mbx);
> + /* Wait for previous message to be acked, timeout 2sec */
> + while (!cptvf->pf_acked) {
> + if (cptvf->pf_nacked)
> + return -EINVAL;
> + msleep(sleep);
> + if (cptvf->pf_acked)
> + break;
> + timeout -= sleep;
> + if (!timeout) {
> + dev_err(&cptvf->pdev->dev, "PF didn't ack to mbox msg %llx from VF%u\n",
> + (mbx->msg & 0xFF), cptvf->vfid);
> + return -EBUSY;
> + }
> + }
> +
> + return 0;
> +}
> +
> +/*
> + * Checks if VF is able to comminicate with PF
> + * and also gets the CPT number this VF is associated to.
> + */
> +int cptvf_check_pf_ready(struct cpt_vf *cptvf)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + struct cpt_mbox mbx = {};
> +
> + mbx.msg = CPT_MSG_READY;
> + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
> + dev_err(&pdev->dev, "PF didn't respond to READY msg\n");
> + return -EBUSY;
> + }
> +
> + return 0;
> +}
> +
> +/*
> + * Communicate VQs size to PF to program CPT(0)_PF_Q(0-15)_CTL of the VF.
> + * Must be ACKed.
> + */
> +int cptvf_send_vq_size_msg(struct cpt_vf *cptvf)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + struct cpt_mbox mbx = {};
> +
> + mbx.msg = CPT_MSG_QLEN;
> + mbx.data = cptvf->qsize;
> + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
> + dev_err(&pdev->dev, "PF didn't respond to vq_size msg\n");
> + return -EBUSY;
> + }
> +
> + return 0;
> +}
> +
> +/*
> + * Communicate VF group required to PF and get the VQ binded to that group
> + */
> +int cptvf_send_vf_to_grp_msg(struct cpt_vf *cptvf)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + struct cpt_mbox mbx = {};
> +
> + mbx.msg = CPT_MSG_QBIND_GRP;
> + /* Convey group of the VF */
> + mbx.data = cptvf->vfgrp;
> + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
> + dev_err(&pdev->dev, "PF didn't respond to vf_type msg\n");
> + return -EBUSY;
> + }
> +
> + return 0;
> +}
> +
> +/*
> + * Communicate VF group required to PF and get the VQ binded to that group
> + */
> +int cptvf_send_vf_priority_msg(struct cpt_vf *cptvf)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + struct cpt_mbox mbx = {};
> +
> + mbx.msg = CPT_MSG_VQ_PRIORITY;
> + /* Convey group of the VF */
> + mbx.data = cptvf->priority;
> + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
> + dev_err(&pdev->dev, "PF didn't respond to vf_type msg\n");
> + return -EBUSY;
> + }
> + return 0;
> +}
> +
> +/*
> + * Communicate to PF that VF is UP and running
> + */
> +int cptvf_send_vf_up(struct cpt_vf *cptvf)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + struct cpt_mbox mbx = {};
> +
> + mbx.msg = CPT_MSG_VF_UP;
> + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
> + dev_err(&pdev->dev, "PF didn't respond to UP msg\n");
> + return -EBUSY;
> + }
> +
> + return 0;
> +}
> +
> +/*
> + * Communicate to PF that VF is DOWN and running
> + */
> +int cptvf_send_vf_down(struct cpt_vf *cptvf)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + struct cpt_mbox mbx = {};
> +
> + mbx.msg = CPT_MSG_VF_DOWN;
> + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
> + dev_err(&pdev->dev, "PF didn't respond to DOWN msg\n");
> + return -EBUSY;
> + }
> +
> + return 0;
> +}
> diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
> new file mode 100644
> index 0000000..062b8e9
> --- /dev/null
> +++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
> @@ -0,0 +1,593 @@
> +/*
> + * Copyright (C) 2016 Cavium, Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of version 2 of the GNU General Public License
> + * as published by the Free Software Foundation.
> + */
> +
> +#include "cptvf.h"
> +#include "request_manager.h"
> +
> +/**
> + * get_free_pending_entry - get free entry from pending queue
> + * @param pqinfo: pending_qinfo structure
> + * @param qno: queue number
> + */
> +static struct pending_entry *get_free_pending_entry(struct pending_queue *q,
> + int qlen)
> +{
> + struct pending_entry *ent = NULL;
> +
> + ent = &q->head[q->rear];
> + if (unlikely(ent->busy)) {
> + ent = NULL;
> + goto no_free_entry;
> + }
> +
> + q->rear++;
> + if (unlikely(q->rear == qlen))
> + q->rear = 0;
> +
> +no_free_entry:
> + return ent;
> +}
> +
> +static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo,
> + int qno)
> +{
> + struct pending_queue *queue = &pqinfo->queue[qno];
> +
> + queue->front++;
> + if (unlikely(queue->front == pqinfo->qlen))
> + queue->front = 0;
> +}
> +
> +static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list,
> + int buf_count, u8 *buffer)
> +{
> + int ret = 0, i, j;
> + int components;
> + struct sglist_component *sg_ptr = NULL;
> + struct pci_dev *pdev = cptvf->pdev;
> +
> + if (unlikely(!list)) {
> + dev_err(&pdev->dev, "Input List pointer is NULL\n");
> + return -EFAULT;
> + }
> +
> + for (i = 0; i < buf_count; i++) {
> + if (likely(list[i].vptr)) {
> + list[i].dma_addr = dma_map_single(&pdev->dev,
> + list[i].vptr,
> + list[i].size,
> + DMA_BIDIRECTIONAL);
> + if (unlikely(dma_mapping_error(&pdev->dev,
> + list[i].dma_addr))) {
> + dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n",
> + i);
> + ret = -EIO;
> + goto sg_cleanup;
> + }
> + }
> + }
> +
> + components = buf_count / 4;
> + sg_ptr = (struct sglist_component *)buffer;
> + for (i = 0; i < components; i++) {
> + sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
> + sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
> + sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
> + sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
> + sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
> + sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
> + sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
> + sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
> + sg_ptr++;
> + }
> +
> + components = buf_count % 4;
> +
> + switch (components) {
> + case 3:
> + sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
> + sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
> + /* Fall through */
> + case 2:
> + sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
> + sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
> + /* Fall through */
> + case 1:
> + sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
> + sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
> + break;
> + default:
> + break;
> + }
> +
> + return ret;
> +
> +sg_cleanup:
> + for (j = 0; j < i; j++) {
> + if (list[j].dma_addr) {
> + dma_unmap_single(&pdev->dev, list[i].dma_addr,
> + list[i].size, DMA_BIDIRECTIONAL);
> + }
> +
> + list[j].dma_addr = 0;
> + }
> +
> + return ret;
> +}
> +
> +static inline int setup_sgio_list(struct cpt_vf *cptvf,
> + struct cpt_info_buffer *info,
> + struct cpt_request_info *req)
> +{
> + u16 g_sz_bytes = 0, s_sz_bytes = 0;
> + int ret = 0;
> + struct pci_dev *pdev = cptvf->pdev;
> +
> + if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) {
> + dev_err(&pdev->dev, "Request SG components are higher than supported\n");
> + ret = -EINVAL;
> + goto scatter_gather_clean;
> + }
> +
> + /* Setup gather (input) components */
> + g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component);
> + info->gather_components = kzalloc(g_sz_bytes, GFP_KERNEL);
> + if (!info->gather_components) {
> + ret = -ENOMEM;
> + goto scatter_gather_clean;
> + }
> +
> + ret = setup_sgio_components(cptvf, req->in,
> + req->incnt,
> + info->gather_components);
> + if (ret) {
> + dev_err(&pdev->dev, "Failed to setup gather list\n");
> + ret = -EFAULT;
> + goto scatter_gather_clean;
> + }
> +
> + /* Setup scatter (output) components */
> + s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component);
> + info->scatter_components = kzalloc(s_sz_bytes, GFP_KERNEL);
> + if (!info->scatter_components) {
> + ret = -ENOMEM;
> + goto scatter_gather_clean;
> + }
> +
> + ret = setup_sgio_components(cptvf, req->out,
> + req->outcnt,
> + info->scatter_components);
> + if (ret) {
> + dev_err(&pdev->dev, "Failed to setup gather list\n");
> + ret = -EFAULT;
> + goto scatter_gather_clean;
> + }
> +
> + /* Create and initialize DPTR */
> + info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
> + info->in_buffer = kzalloc(info->dlen, GFP_KERNEL);
> + if (!info->in_buffer) {
> + ret = -ENOMEM;
> + goto scatter_gather_clean;
> + }
> +
> + ((u16 *)info->in_buffer)[0] = req->outcnt;
> + ((u16 *)info->in_buffer)[1] = req->incnt;
> + ((u16 *)info->in_buffer)[2] = 0;
> + ((u16 *)info->in_buffer)[3] = 0;
> + *(u64 *)info->in_buffer = cpu_to_be64p((u64 *)info->in_buffer);
> +
> + memcpy(&info->in_buffer[8], info->gather_components,
> + g_sz_bytes);
> + memcpy(&info->in_buffer[8 + g_sz_bytes],
> + info->scatter_components, s_sz_bytes);
> +
> + info->dptr_baddr = dma_map_single(&pdev->dev,
> + (void *)info->in_buffer,
> + info->dlen,
> + DMA_BIDIRECTIONAL);
> + if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) {
> + dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen);
> + ret = -EIO;
> + goto scatter_gather_clean;
> + }
> +
> + /* Create and initialize RPTR */
> + info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, GFP_KERNEL);
> + if (!info->out_buffer) {
> + ret = -ENOMEM;
> + goto scatter_gather_clean;
> + }
> +
> + *((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT);
> + info->alternate_caddr = (u64 *)info->out_buffer;
> + info->rptr_baddr = dma_map_single(&pdev->dev,
> + (void *)info->out_buffer,
> + COMPLETION_CODE_SIZE,
> + DMA_BIDIRECTIONAL);
> + if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) {
> + dev_err(&pdev->dev, "Mapping RPTR Failed %d\n",
> + COMPLETION_CODE_SIZE);
> + ret = -EIO;
> + goto scatter_gather_clean;
> + }
> +
> + return 0;
> +
> +scatter_gather_clean:
> + return ret;
> +}
> +
> +int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd,
> + u32 qno)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + struct command_qinfo *qinfo = NULL;
> + struct command_queue *queue;
> + struct command_chunk *chunk;
> + u8 *ent;
> + int ret = 0;
> +
> + if (unlikely(qno >= cptvf->nr_queues)) {
> + dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n",
> + qno, cptvf->nr_queues);
> + return -EINVAL;
> + }
> +
> + qinfo = &cptvf->cqinfo;
> + queue = &qinfo->queue[qno];
> + /* lock commad queue */
> + spin_lock(&queue->lock);
> + ent = &queue->qhead->head[queue->idx * qinfo->cmd_size];
> + memcpy(ent, (void *)cmd, qinfo->cmd_size);
> +
> + if (++queue->idx >= queue->qhead->size / 64) {
> + struct hlist_node *node;
> +
> + hlist_for_each(node, &queue->chead) {
> + chunk = hlist_entry(node, struct command_chunk,
> + nextchunk);
> + if (chunk == queue->qhead) {
> + continue;
> + } else {
> + queue->qhead = chunk;
> + break;
> + }
> + }
> + queue->idx = 0;
> + }
> + /* make sure all memory stores are done before ringing doorbell */
> + smp_wmb();
> + cptvf_write_vq_doorbell(cptvf, 1);
> + /* unlock command queue */
> + spin_unlock(&queue->lock);
> +
> + return ret;
> +}
> +
> +void do_request_cleanup(struct cpt_vf *cptvf,
> + struct cpt_info_buffer *info)
> +{
> + int i;
> + struct pci_dev *pdev = cptvf->pdev;
> + struct cpt_request_info *req;
> +
> + if (info->dptr_baddr)
> + dma_unmap_single(&pdev->dev, info->dptr_baddr,
> + info->dlen, DMA_BIDIRECTIONAL);
> +
> + if (info->rptr_baddr)
> + dma_unmap_single(&pdev->dev, info->rptr_baddr,
> + COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL);
> +
> + if (info->comp_baddr)
> + dma_unmap_single(&pdev->dev, info->comp_baddr,
> + sizeof(union cpt_res_s), DMA_BIDIRECTIONAL);
> +
> + if (info->req) {
> + req = info->req;
> + for (i = 0; i < req->outcnt; i++) {
> + if (req->out[i].dma_addr)
> + dma_unmap_single(&pdev->dev,
> + req->out[i].dma_addr,
> + req->out[i].size,
> + DMA_BIDIRECTIONAL);
> + }
> +
> + for (i = 0; i < req->incnt; i++) {
> + if (req->in[i].dma_addr)
> + dma_unmap_single(&pdev->dev,
> + req->in[i].dma_addr,
> + req->in[i].size,
> + DMA_BIDIRECTIONAL);
> + }
> + }
> +
> + if (info->scatter_components)
> + kzfree(info->scatter_components);
> +
> + if (info->gather_components)
> + kzfree(info->gather_components);
> +
> + if (info->out_buffer)
> + kzfree(info->out_buffer);
> +
> + if (info->in_buffer)
> + kzfree(info->in_buffer);
> +
> + if (info->completion_addr)
> + kzfree((void *)info->completion_addr);
> +
> + kzfree(info);
> +}
> +
> +void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> +
> + if (!info || !cptvf) {
> + dev_err(&pdev->dev, "Input params are incorrect for post processing\n");
> + return;
> + }
> +
> + do_request_cleanup(cptvf, info);
> +}
> +
> +static inline void process_pending_queue(struct cpt_vf *cptvf,
> + struct pending_qinfo *pqinfo,
> + int qno)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> + struct pending_queue *pqueue = &pqinfo->queue[qno];
> + struct pending_entry *pentry = NULL;
> + struct cpt_info_buffer *info = NULL;
> + union cpt_res_s *status = NULL;
> + unsigned char ccode;
> +
> + while (1) {
> + spin_lock_bh(&pqueue->lock);
> + pentry = &pqueue->head[pqueue->front];
> + if (unlikely(!pentry->busy)) {
> + spin_unlock_bh(&pqueue->lock);
> + break;
> + }
> +
> + info = (struct cpt_info_buffer *)pentry->post_arg;
> + if (unlikely(!info)) {
> + dev_err(&pdev->dev, "Pending Entry post arg NULL\n");
> + pending_queue_inc_front(pqinfo, qno);
> + spin_unlock_bh(&pqueue->lock);
> + continue;
> + }
> +
> + status = (union cpt_res_s *)pentry->completion_addr;
> + ccode = status->s.compcode;
> + if ((status->s.compcode == CPT_COMP_E_FAULT) ||
> + (status->s.compcode == CPT_COMP_E_SWERR)) {
> + dev_err(&pdev->dev, "Request failed with %s\n",
> + (status->s.compcode == CPT_COMP_E_FAULT) ?
> + "DMA Fault" : "Software error");
> + pentry->completion_addr = NULL;
> + pentry->busy = false;
> + atomic64_dec((&pqueue->pending_count));
> + pentry->post_arg = NULL;
> + pending_queue_inc_front(pqinfo, qno);
> + do_request_cleanup(cptvf, info);
> + spin_unlock_bh(&pqueue->lock);
> + break;
> + } else if (status->s.compcode == COMPLETION_CODE_INIT) {
> + /* check for timeout */
> + if (time_after_eq(jiffies,
> + (info->time_in +
> + (CPT_COMMAND_TIMEOUT * HZ)))) {
> + dev_err(&pdev->dev, "Request timed out");
> + pentry->completion_addr = NULL;
> + pentry->busy = false;
> + atomic64_dec((&pqueue->pending_count));
> + pentry->post_arg = NULL;
> + pending_queue_inc_front(pqinfo, qno);
> + do_request_cleanup(cptvf, info);
> + spin_unlock_bh(&pqueue->lock);
> + break;
> + } else if ((*info->alternate_caddr ==
> + (~COMPLETION_CODE_INIT)) &&
> + (info->extra_time < TIME_IN_RESET_COUNT)) {
> + info->time_in = jiffies;
> + info->extra_time++;
> + spin_unlock_bh(&pqueue->lock);
> + break;
> + }
> + }
> +
> + pentry->completion_addr = NULL;
> + pentry->busy = false;
> + pentry->post_arg = NULL;
> + atomic64_dec((&pqueue->pending_count));
> + pending_queue_inc_front(pqinfo, qno);
> + spin_unlock_bh(&pqueue->lock);
> +
> + do_post_process(info->cptvf, info);
> + /*
> + * Calling callback after we find
> + * that the request has been serviced
> + */
> + pentry->callback(ccode, pentry->callback_arg);
> + }
> +}
> +
> +int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req)
> +{
> + int ret = 0, clear = 0, queue = 0;
> + struct cpt_info_buffer *info = NULL;
> + struct cptvf_request *cpt_req = NULL;
> + union ctrl_info *ctrl = NULL;
> + struct pending_entry *pentry = NULL;
> + struct pending_queue *pqueue = NULL;
> + struct pci_dev *pdev = cptvf->pdev;
> + u8 group = 0;
> + struct cpt_vq_command vq_cmd;
> + union cpt_inst_s cptinst;
> +
> + if (unlikely(!cptvf || !req)) {
You already dereferenced cptvf above.
> + dev_err(&pdev->dev, "Invalid inputs (cptvf: %p, req: %p)\n",
> + cptvf, req);
> + return -EINVAL;
> + }
> +
> + info = kzalloc(sizeof(*info), GFP_KERNEL | GFP_ATOMIC);
What do you expect to happen with GFP_KERNEL | GFP_ATOMIC?
> + if (unlikely(!info)) {
> + dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n");
> + return -ENOMEM;
> + }
> +
> + cpt_req = (struct cptvf_request *)&req->req;
> + ctrl = (union ctrl_info *)&req->ctrl;
> +
> + info->cptvf = cptvf;
> + group = ctrl->s.grp;
> + ret = setup_sgio_list(cptvf, info, req);
> + if (ret) {
> + dev_err(&pdev->dev, "Setting up SG list failed");
> + goto request_cleanup;
> + }
> +
> + cpt_req->dlen = info->dlen;
> + /*
> + * Get buffer for union cpt_res_s response
> + * structure and its physical address
> + */
> + info->completion_addr = kzalloc(sizeof(union cpt_res_s),
> + GFP_KERNEL | GFP_ATOMIC);
Same as above, you also never checked if it had failed.
> + *((u8 *)(info->completion_addr)) = COMPLETION_CODE_INIT;
Supposedly info->completion_addr is a "union cpt_res_s", why do you
cast it to u8 ptr?
> + info->comp_baddr = dma_map_single(&pdev->dev,
> + (void *)info->completion_addr,
> + sizeof(union cpt_res_s),
> + DMA_BIDIRECTIONAL);
> + if (dma_mapping_error(&pdev->dev, info->comp_baddr)) {
> + dev_err(&pdev->dev, "mapping compptr Failed %lu\n",
> + sizeof(union cpt_res_s));
> + ret = -EFAULT;
> + goto request_cleanup;
> + }
> +
> + /* Fill the VQ command */
> + vq_cmd.cmd.u64 = 0;
> + vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
> + vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
> + vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
> + vq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen);
> +
> + /* 64-bit swap for microcode data reads, not needed for addresses*/
> + vq_cmd.cmd.u64 = cpu_to_be64(vq_cmd.cmd.u64);
> + vq_cmd.dptr = info->dptr_baddr;
> + vq_cmd.rptr = info->rptr_baddr;
> + vq_cmd.cptr.u64 = 0;
> + vq_cmd.cptr.s.grp = group;
> + /* Get Pending Entry to submit command */
> + /* Always queue 0, because 1 queue per VF */
> + queue = 0;
> + pqueue = &cptvf->pqinfo.queue[queue];
> +
> + if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) {
> + dev_err(&pdev->dev, "pending threshold reached\n");
> + process_pending_queue(cptvf, &cptvf->pqinfo, queue);
> + }
> +
> +get_pending_entry:
> + spin_lock_bh(&pqueue->lock);
> + pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen);
> + if (unlikely(!pentry)) {
> + spin_unlock_bh(&pqueue->lock);
> + if (clear == 0) {
> + process_pending_queue(cptvf, &cptvf->pqinfo, queue);
> + clear = 1;
> + goto get_pending_entry;
> + }
> + dev_err(&pdev->dev, "Get free entry failed\n");
> + dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n",
> + queue, pqueue->rear, pqueue->front);
> + ret = -EFAULT;
> + goto request_cleanup;
> + }
> +
> + pentry->completion_addr = info->completion_addr;
> + pentry->post_arg = (void *)info;
> + pentry->callback = req->callback;
> + pentry->callback_arg = req->callback_arg;
> + info->pentry = pentry;
> + pentry->busy = true;
> + atomic64_inc(&pqueue->pending_count);
> +
> + /* Send CPT command */
> + info->pentry = pentry;
> + info->time_in = jiffies;
> + info->req = req;
> +
> + /* Create the CPT_INST_S type command for HW intrepretation */
> + cptinst.s.doneint = true;
> + cptinst.s.res_addr = (u64)info->comp_baddr;
> + cptinst.s.tag = 0;
> + cptinst.s.grp = 0;
> + cptinst.s.wq_ptr = 0;
> + cptinst.s.ei0 = vq_cmd.cmd.u64;
> + cptinst.s.ei1 = vq_cmd.dptr;
> + cptinst.s.ei2 = vq_cmd.rptr;
> + cptinst.s.ei3 = vq_cmd.cptr.u64;
> +
> + ret = send_cpt_command(cptvf, &cptinst, queue);
> + spin_unlock_bh(&pqueue->lock);
> + if (unlikely(ret)) {
> + dev_err(&pdev->dev, "Send command failed for AE\n");
> + ret = -EFAULT;
> + goto request_cleanup;
> + }
> +
> + return 0;
> +
> +request_cleanup:
> + dev_dbg(&pdev->dev, "Failed to submit CPT command\n");
> + do_request_cleanup(cptvf, info);
> +
> + return ret;
> +}
> +
> +void vq_post_process(struct cpt_vf *cptvf, u32 qno)
> +{
> + struct pci_dev *pdev = cptvf->pdev;
> +
> + if (unlikely(qno > cptvf->nr_queues)) {
> + dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n",
> + qno);
> + return;
> + }
> +
> + process_pending_queue(cptvf, &cptvf->pqinfo, qno);
> +}
> +
> +int cptvf_do_request(void *vfdev, struct cpt_request_info *req)
> +{
> + struct cpt_vf *cptvf = (struct cpt_vf *)vfdev;
> + struct pci_dev *pdev = cptvf->pdev;
> +
> + if (!cpt_device_ready(cptvf)) {
> + dev_err(&pdev->dev, "CPT Device is not ready");
> + return -ENODEV;
> + }
> +
> + if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) {
> + dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request",
> + cptvf->vfid);
> + return -EINVAL;
> + } else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) {
> + dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request",
> + cptvf->vfid);
> + return -EINVAL;
> + }
> +
> + return process_request(cptvf, req);
> +}
^ permalink raw reply
* Re: [PATCH] crypto: ccp: Fix double add when creating new DMA command
From: Gary R Hook @ 2017-02-02 17:15 UTC (permalink / raw)
To: linux-crypto@vger.kernel.org
Cc: herbert@gondor.apana.org.au, davem@davemloft.net
In-Reply-To: <20170127230904.31399.58184.stgit@taos>
On 01/27/2017 05:09 PM, Gary R Hook wrote:
> Eliminate a double-add by creating a new list to manage
> command descriptors when created; move the descriptor to
> the pending list when the command is submitted.
Herbert,
Another patch that could use some 4.10 love. Possible?
Thanks,
Gary
>
>
> Signed-off-by: Gary R Hook <gary.hook@amd.com>
> ---
> drivers/crypto/ccp/ccp-dev.h | 1 +
> drivers/crypto/ccp/ccp-dmaengine.c | 6 +++++-
> 2 files changed, 6 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h
> index 830f35e..649e561 100644
> --- a/drivers/crypto/ccp/ccp-dev.h
> +++ b/drivers/crypto/ccp/ccp-dev.h
> @@ -238,6 +238,7 @@ struct ccp_dma_chan {
> struct ccp_device *ccp;
>
> spinlock_t lock;
> + struct list_head created;
> struct list_head pending;
> struct list_head active;
> struct list_head complete;
> diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c
> index 6553912..e5d9278 100644
> --- a/drivers/crypto/ccp/ccp-dmaengine.c
> +++ b/drivers/crypto/ccp/ccp-dmaengine.c
> @@ -63,6 +63,7 @@ static void ccp_free_chan_resources(struct dma_chan *dma_chan)
> ccp_free_desc_resources(chan->ccp, &chan->complete);
> ccp_free_desc_resources(chan->ccp, &chan->active);
> ccp_free_desc_resources(chan->ccp, &chan->pending);
> + ccp_free_desc_resources(chan->ccp, &chan->created);
>
> spin_unlock_irqrestore(&chan->lock, flags);
> }
> @@ -273,6 +274,7 @@ static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc)
> spin_lock_irqsave(&chan->lock, flags);
>
> cookie = dma_cookie_assign(tx_desc);
> + list_del(&desc->entry);
> list_add_tail(&desc->entry, &chan->pending);
>
> spin_unlock_irqrestore(&chan->lock, flags);
> @@ -426,7 +428,7 @@ static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan,
>
> spin_lock_irqsave(&chan->lock, sflags);
>
> - list_add_tail(&desc->entry, &chan->pending);
> + list_add_tail(&desc->entry, &chan->created);
>
> spin_unlock_irqrestore(&chan->lock, sflags);
>
> @@ -610,6 +612,7 @@ static int ccp_terminate_all(struct dma_chan *dma_chan)
> /*TODO: Purge the complete list? */
> ccp_free_desc_resources(chan->ccp, &chan->active);
> ccp_free_desc_resources(chan->ccp, &chan->pending);
> + ccp_free_desc_resources(chan->ccp, &chan->created);
>
> spin_unlock_irqrestore(&chan->lock, flags);
>
> @@ -679,6 +682,7 @@ int ccp_dmaengine_register(struct ccp_device *ccp)
> chan->ccp = ccp;
>
> spin_lock_init(&chan->lock);
> + INIT_LIST_HEAD(&chan->created);
> INIT_LIST_HEAD(&chan->pending);
> INIT_LIST_HEAD(&chan->active);
> INIT_LIST_HEAD(&chan->complete);
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
--
This is my day job. Follow me at:
IG/Twitter/Facebook: @grhookphoto
IG/Twitter/Facebook: @grhphotographer
^ permalink raw reply
* Re: [PATCH] crypto: ccp: Fix DMA operations when IOMMU is enabled
From: Gary R Hook @ 2017-02-02 17:14 UTC (permalink / raw)
To: linux-crypto@vger.kernel.org
Cc: herbert@gondor.apana.org.au, davem@davemloft.net
In-Reply-To: <20170127212845.27618.32169.stgit@taos>
On 01/27/2017 03:28 PM, Gary R Hook wrote:
> An I/O page fault occurs when the IOMMU is enabled on a
> system that supports the v5 CCP. DMA operations use a
> Request ID value that does not match what is expected by
> the IOMMU, resulting in the I/O page fault. Setting the
> Request ID value to 0 corrects this issue.
Herbert,
Hoping to get some 4.10 love on this patch, plus one other. Do we see
a possibility at this late date?
>
> Signed-off-by: Gary R Hook <gary.hook@amd.com>
> ---
> drivers/crypto/ccp/ccp-dev-v5.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c
> index e2ce819..612898b 100644
> --- a/drivers/crypto/ccp/ccp-dev-v5.c
> +++ b/drivers/crypto/ccp/ccp-dev-v5.c
> @@ -959,7 +959,7 @@ static irqreturn_t ccp5_irq_handler(int irq, void *data)
> static void ccp5_config(struct ccp_device *ccp)
> {
> /* Public side */
> - iowrite32(0x00001249, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);
> + iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);
> }
>
> static void ccp5other_config(struct ccp_device *ccp)
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
--
This is my day job. Follow me at:
IG/Twitter/Facebook: @grhookphoto
IG/Twitter/Facebook: @grhphotographer
^ permalink raw reply
* Re: [PATCH] crypto: arm64/crc32 - merge CRC32 and PMULL instruction based drivers
From: Matthias Brugger @ 2017-02-02 16:47 UTC (permalink / raw)
To: Ard Biesheuvel, linux-arm-kernel, linux-crypto
Cc: will.deacon, herbert, yazen.ghannam, steve.capper, agraf
In-Reply-To: <1485963340-25297-1-git-send-email-ard.biesheuvel@linaro.org>
On 02/01/2017 04:35 PM, Ard Biesheuvel wrote:
> The PMULL based CRC32 implementation already contains code based on the
> separate, optional CRC32 instructions to fallback to when operating on
> small quantities of data. We can expose these routines directly on systems
> that lack the 64x64 PMULL instructions but do implement the CRC32 ones,
> which makes the driver that is based solely on those CRC32 instructions
> redundant. So remove it.
>
> Note that this aligns arm64 with ARM, whose accelerated CRC32 driver
> also combines the CRC32 extension based and the PMULL based versions.
>
> Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
> ---
>
> This is a meaningful patch by itself imho, but also fixes the issue reported
> by Matthias where their v4.8 based GCC does not understand the -mcpu=generic+crc
> command line option, resulting in failed builds.
>
This patch fixes the issue, thanks.
Tested-by: Matthias Brugger <mbrugger@suse.com>
> arch/arm64/configs/defconfig | 1 -
> arch/arm64/crypto/Kconfig | 9 +-
> arch/arm64/crypto/Makefile | 4 -
> arch/arm64/crypto/crc32-arm64.c | 290 --------------------
> arch/arm64/crypto/crc32-ce-glue.c | 49 +++-
> 5 files changed, 41 insertions(+), 312 deletions(-)
>
> diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
> index 33b744d54739..6fc6f5a2a6e5 100644
> --- a/arch/arm64/configs/defconfig
> +++ b/arch/arm64/configs/defconfig
> @@ -516,4 +516,3 @@ CONFIG_CRYPTO_GHASH_ARM64_CE=y
> CONFIG_CRYPTO_AES_ARM64_CE_CCM=y
> CONFIG_CRYPTO_AES_ARM64_CE_BLK=y
> # CONFIG_CRYPTO_AES_ARM64_NEON_BLK is not set
> -CONFIG_CRYPTO_CRC32_ARM64=y
> diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
> index bed7feddfeed..d92293747d63 100644
> --- a/arch/arm64/crypto/Kconfig
> +++ b/arch/arm64/crypto/Kconfig
> @@ -37,8 +37,8 @@ config CRYPTO_CRCT10DIF_ARM64_CE
> select CRYPTO_HASH
>
> config CRYPTO_CRC32_ARM64_CE
> - tristate "CRC32 and CRC32C digest algorithms using PMULL instructions"
> - depends on KERNEL_MODE_NEON && CRC32
> + tristate "CRC32 and CRC32C digest algorithms using ARMv8 extensions"
> + depends on CRC32
> select CRYPTO_HASH
>
> config CRYPTO_AES_ARM64
> @@ -71,11 +71,6 @@ config CRYPTO_AES_ARM64_NEON_BLK
> select CRYPTO_AES
> select CRYPTO_SIMD
>
> -config CRYPTO_CRC32_ARM64
> - tristate "CRC32 and CRC32C using optional ARMv8 instructions"
> - depends on ARM64
> - select CRYPTO_HASH
> -
> config CRYPTO_CHACHA20_NEON
> tristate "NEON accelerated ChaCha20 symmetric cipher"
> depends on KERNEL_MODE_NEON
> diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
> index d1ae1b9cbe70..b5edc5918c28 100644
> --- a/arch/arm64/crypto/Makefile
> +++ b/arch/arm64/crypto/Makefile
> @@ -55,10 +55,6 @@ AFLAGS_aes-neon.o := -DINTERLEAVE=4
>
> CFLAGS_aes-glue-ce.o := -DUSE_V8_CRYPTO_EXTENSIONS
>
> -obj-$(CONFIG_CRYPTO_CRC32_ARM64) += crc32-arm64.o
> -
> -CFLAGS_crc32-arm64.o := -mcpu=generic+crc
> -
> $(obj)/aes-glue-%.o: $(src)/aes-glue.c FORCE
> $(call if_changed_rule,cc_o_c)
>
> diff --git a/arch/arm64/crypto/crc32-arm64.c b/arch/arm64/crypto/crc32-arm64.c
> deleted file mode 100644
> index 6a37c3c6b11d..000000000000
> --- a/arch/arm64/crypto/crc32-arm64.c
> +++ /dev/null
> @@ -1,290 +0,0 @@
> -/*
> - * crc32-arm64.c - CRC32 and CRC32C using optional ARMv8 instructions
> - *
> - * Module based on crypto/crc32c_generic.c
> - *
> - * CRC32 loop taken from Ed Nevill's Hadoop CRC patch
> - * http://mail-archives.apache.org/mod_mbox/hadoop-common-dev/201406.mbox/%3C1403687030.3355.19.camel%40localhost.localdomain%3E
> - *
> - * Using inline assembly instead of intrinsics in order to be backwards
> - * compatible with older compilers.
> - *
> - * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
> - *
> - * This program is free software; you can redistribute it and/or modify
> - * it under the terms of the GNU General Public License version 2 as
> - * published by the Free Software Foundation.
> - */
> -
> -#include <linux/unaligned/access_ok.h>
> -#include <linux/cpufeature.h>
> -#include <linux/init.h>
> -#include <linux/kernel.h>
> -#include <linux/module.h>
> -#include <linux/string.h>
> -
> -#include <crypto/internal/hash.h>
> -
> -MODULE_AUTHOR("Yazen Ghannam <yazen.ghannam@linaro.org>");
> -MODULE_DESCRIPTION("CRC32 and CRC32C using optional ARMv8 instructions");
> -MODULE_LICENSE("GPL v2");
> -
> -#define CRC32X(crc, value) __asm__("crc32x %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))
> -#define CRC32W(crc, value) __asm__("crc32w %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
> -#define CRC32H(crc, value) __asm__("crc32h %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
> -#define CRC32B(crc, value) __asm__("crc32b %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
> -#define CRC32CX(crc, value) __asm__("crc32cx %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))
> -#define CRC32CW(crc, value) __asm__("crc32cw %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
> -#define CRC32CH(crc, value) __asm__("crc32ch %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
> -#define CRC32CB(crc, value) __asm__("crc32cb %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
> -
> -static u32 crc32_arm64_le_hw(u32 crc, const u8 *p, unsigned int len)
> -{
> - s64 length = len;
> -
> - while ((length -= sizeof(u64)) >= 0) {
> - CRC32X(crc, get_unaligned_le64(p));
> - p += sizeof(u64);
> - }
> -
> - /* The following is more efficient than the straight loop */
> - if (length & sizeof(u32)) {
> - CRC32W(crc, get_unaligned_le32(p));
> - p += sizeof(u32);
> - }
> - if (length & sizeof(u16)) {
> - CRC32H(crc, get_unaligned_le16(p));
> - p += sizeof(u16);
> - }
> - if (length & sizeof(u8))
> - CRC32B(crc, *p);
> -
> - return crc;
> -}
> -
> -static u32 crc32c_arm64_le_hw(u32 crc, const u8 *p, unsigned int len)
> -{
> - s64 length = len;
> -
> - while ((length -= sizeof(u64)) >= 0) {
> - CRC32CX(crc, get_unaligned_le64(p));
> - p += sizeof(u64);
> - }
> -
> - /* The following is more efficient than the straight loop */
> - if (length & sizeof(u32)) {
> - CRC32CW(crc, get_unaligned_le32(p));
> - p += sizeof(u32);
> - }
> - if (length & sizeof(u16)) {
> - CRC32CH(crc, get_unaligned_le16(p));
> - p += sizeof(u16);
> - }
> - if (length & sizeof(u8))
> - CRC32CB(crc, *p);
> -
> - return crc;
> -}
> -
> -#define CHKSUM_BLOCK_SIZE 1
> -#define CHKSUM_DIGEST_SIZE 4
> -
> -struct chksum_ctx {
> - u32 key;
> -};
> -
> -struct chksum_desc_ctx {
> - u32 crc;
> -};
> -
> -static int chksum_init(struct shash_desc *desc)
> -{
> - struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
> - struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
> -
> - ctx->crc = mctx->key;
> -
> - return 0;
> -}
> -
> -/*
> - * Setting the seed allows arbitrary accumulators and flexible XOR policy
> - * If your algorithm starts with ~0, then XOR with ~0 before you set
> - * the seed.
> - */
> -static int chksum_setkey(struct crypto_shash *tfm, const u8 *key,
> - unsigned int keylen)
> -{
> - struct chksum_ctx *mctx = crypto_shash_ctx(tfm);
> -
> - if (keylen != sizeof(mctx->key)) {
> - crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
> - return -EINVAL;
> - }
> - mctx->key = get_unaligned_le32(key);
> - return 0;
> -}
> -
> -static int chksum_update(struct shash_desc *desc, const u8 *data,
> - unsigned int length)
> -{
> - struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
> -
> - ctx->crc = crc32_arm64_le_hw(ctx->crc, data, length);
> - return 0;
> -}
> -
> -static int chksumc_update(struct shash_desc *desc, const u8 *data,
> - unsigned int length)
> -{
> - struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
> -
> - ctx->crc = crc32c_arm64_le_hw(ctx->crc, data, length);
> - return 0;
> -}
> -
> -static int chksum_final(struct shash_desc *desc, u8 *out)
> -{
> - struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
> -
> - put_unaligned_le32(ctx->crc, out);
> - return 0;
> -}
> -
> -static int chksumc_final(struct shash_desc *desc, u8 *out)
> -{
> - struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
> -
> - put_unaligned_le32(~ctx->crc, out);
> - return 0;
> -}
> -
> -static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
> -{
> - put_unaligned_le32(crc32_arm64_le_hw(crc, data, len), out);
> - return 0;
> -}
> -
> -static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
> -{
> - put_unaligned_le32(~crc32c_arm64_le_hw(crc, data, len), out);
> - return 0;
> -}
> -
> -static int chksum_finup(struct shash_desc *desc, const u8 *data,
> - unsigned int len, u8 *out)
> -{
> - struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
> -
> - return __chksum_finup(ctx->crc, data, len, out);
> -}
> -
> -static int chksumc_finup(struct shash_desc *desc, const u8 *data,
> - unsigned int len, u8 *out)
> -{
> - struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
> -
> - return __chksumc_finup(ctx->crc, data, len, out);
> -}
> -
> -static int chksum_digest(struct shash_desc *desc, const u8 *data,
> - unsigned int length, u8 *out)
> -{
> - struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
> -
> - return __chksum_finup(mctx->key, data, length, out);
> -}
> -
> -static int chksumc_digest(struct shash_desc *desc, const u8 *data,
> - unsigned int length, u8 *out)
> -{
> - struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
> -
> - return __chksumc_finup(mctx->key, data, length, out);
> -}
> -
> -static int crc32_cra_init(struct crypto_tfm *tfm)
> -{
> - struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
> -
> - mctx->key = 0;
> - return 0;
> -}
> -
> -static int crc32c_cra_init(struct crypto_tfm *tfm)
> -{
> - struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
> -
> - mctx->key = ~0;
> - return 0;
> -}
> -
> -static struct shash_alg crc32_alg = {
> - .digestsize = CHKSUM_DIGEST_SIZE,
> - .setkey = chksum_setkey,
> - .init = chksum_init,
> - .update = chksum_update,
> - .final = chksum_final,
> - .finup = chksum_finup,
> - .digest = chksum_digest,
> - .descsize = sizeof(struct chksum_desc_ctx),
> - .base = {
> - .cra_name = "crc32",
> - .cra_driver_name = "crc32-arm64-hw",
> - .cra_priority = 300,
> - .cra_blocksize = CHKSUM_BLOCK_SIZE,
> - .cra_alignmask = 0,
> - .cra_ctxsize = sizeof(struct chksum_ctx),
> - .cra_module = THIS_MODULE,
> - .cra_init = crc32_cra_init,
> - }
> -};
> -
> -static struct shash_alg crc32c_alg = {
> - .digestsize = CHKSUM_DIGEST_SIZE,
> - .setkey = chksum_setkey,
> - .init = chksum_init,
> - .update = chksumc_update,
> - .final = chksumc_final,
> - .finup = chksumc_finup,
> - .digest = chksumc_digest,
> - .descsize = sizeof(struct chksum_desc_ctx),
> - .base = {
> - .cra_name = "crc32c",
> - .cra_driver_name = "crc32c-arm64-hw",
> - .cra_priority = 300,
> - .cra_blocksize = CHKSUM_BLOCK_SIZE,
> - .cra_alignmask = 0,
> - .cra_ctxsize = sizeof(struct chksum_ctx),
> - .cra_module = THIS_MODULE,
> - .cra_init = crc32c_cra_init,
> - }
> -};
> -
> -static int __init crc32_mod_init(void)
> -{
> - int err;
> -
> - err = crypto_register_shash(&crc32_alg);
> -
> - if (err)
> - return err;
> -
> - err = crypto_register_shash(&crc32c_alg);
> -
> - if (err) {
> - crypto_unregister_shash(&crc32_alg);
> - return err;
> - }
> -
> - return 0;
> -}
> -
> -static void __exit crc32_mod_exit(void)
> -{
> - crypto_unregister_shash(&crc32_alg);
> - crypto_unregister_shash(&crc32c_alg);
> -}
> -
> -module_cpu_feature_match(CRC32, crc32_mod_init);
> -module_exit(crc32_mod_exit);
> diff --git a/arch/arm64/crypto/crc32-ce-glue.c b/arch/arm64/crypto/crc32-ce-glue.c
> index 8594127d5e01..eccb1ae90064 100644
> --- a/arch/arm64/crypto/crc32-ce-glue.c
> +++ b/arch/arm64/crypto/crc32-ce-glue.c
> @@ -72,6 +72,24 @@ static int crc32_pmull_init(struct shash_desc *desc)
> return 0;
> }
>
> +static int crc32_update(struct shash_desc *desc, const u8 *data,
> + unsigned int length)
> +{
> + u32 *crc = shash_desc_ctx(desc);
> +
> + *crc = crc32_armv8_le(*crc, data, length);
> + return 0;
> +}
> +
> +static int crc32c_update(struct shash_desc *desc, const u8 *data,
> + unsigned int length)
> +{
> + u32 *crc = shash_desc_ctx(desc);
> +
> + *crc = crc32c_armv8_le(*crc, data, length);
> + return 0;
> +}
> +
> static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
> unsigned int length)
> {
> @@ -156,7 +174,7 @@ static int crc32c_pmull_final(struct shash_desc *desc, u8 *out)
> static struct shash_alg crc32_pmull_algs[] = { {
> .setkey = crc32_pmull_setkey,
> .init = crc32_pmull_init,
> - .update = crc32_pmull_update,
> + .update = crc32_update,
> .final = crc32_pmull_final,
> .descsize = sizeof(u32),
> .digestsize = sizeof(u32),
> @@ -171,7 +189,7 @@ static struct shash_alg crc32_pmull_algs[] = { {
> }, {
> .setkey = crc32_pmull_setkey,
> .init = crc32_pmull_init,
> - .update = crc32c_pmull_update,
> + .update = crc32c_update,
> .final = crc32c_pmull_final,
> .descsize = sizeof(u32),
> .digestsize = sizeof(u32),
> @@ -187,14 +205,20 @@ static struct shash_alg crc32_pmull_algs[] = { {
>
> static int __init crc32_pmull_mod_init(void)
> {
> - if (elf_hwcap & HWCAP_CRC32) {
> - fallback_crc32 = crc32_armv8_le;
> - fallback_crc32c = crc32c_armv8_le;
> - } else {
> - fallback_crc32 = crc32_le;
> - fallback_crc32c = __crc32c_le;
> + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_PMULL)) {
> + crc32_pmull_algs[0].update = crc32_pmull_update;
> + crc32_pmull_algs[1].update = crc32c_pmull_update;
> +
> + if (elf_hwcap & HWCAP_CRC32) {
> + fallback_crc32 = crc32_armv8_le;
> + fallback_crc32c = crc32c_armv8_le;
> + } else {
> + fallback_crc32 = crc32_le;
> + fallback_crc32c = __crc32c_le;
> + }
> + } else if (!(elf_hwcap & HWCAP_CRC32)) {
> + return -ENODEV;
> }
> -
> return crypto_register_shashes(crc32_pmull_algs,
> ARRAY_SIZE(crc32_pmull_algs));
> }
> @@ -205,7 +229,12 @@ static void __exit crc32_pmull_mod_exit(void)
> ARRAY_SIZE(crc32_pmull_algs));
> }
>
> -module_cpu_feature_match(PMULL, crc32_pmull_mod_init);
> +static const struct cpu_feature crc32_cpu_feature[] = {
> + { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
> +};
> +MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
> +
> +module_init(crc32_pmull_mod_init);
> module_exit(crc32_pmull_mod_exit);
>
> MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
>
^ permalink raw reply
* [PATCH v3] crypto: aes - add generic time invariant AES cipher
From: Ard Biesheuvel @ 2017-02-02 16:37 UTC (permalink / raw)
To: linux-crypto, herbert, ebiggers3; +Cc: Ard Biesheuvel
Lookup table based AES is sensitive to timing attacks, which is due to
the fact that such table lookups are data dependent, and the fact that
8 KB worth of tables covers a significant number of cachelines on any
architecture, resulting in an exploitable correlation between the key
and the processing time for known plaintexts.
For network facing algorithms such as CTR, CCM or GCM, this presents a
security risk, which is why arch specific AES ports are typically time
invariant, either through the use of special instructions, or by using
SIMD algorithms that don't rely on table lookups.
For generic code, this is difficult to achieve without losing too much
performance, but we can improve the situation significantly by switching
to an implementation that only needs 256 bytes of table data (the actual
S-box itself), which can be prefetched at the start of each block to
eliminate data dependent latencies.
This code encrypts at ~25 cycles per byte on ARM Cortex-A57 (while the
ordinary generic AES driver manages 18 cycles per byte on this
hardware). Decryption is substantially slower.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
Sending this out as a separate patch since the CCM/CBCMAC series that this
was part of has not actually changed, and does not really depend on this
patch at all.
I have included a decrypt() and a setkey() routine, which allows this driver
to be used as a replacement for the generic AES code. However, the lookup
tables exposed by the generic AES driver are used in a couple of other places
as well.
crypto/Kconfig | 17 +
crypto/Makefile | 1 +
crypto/aes_ti.c | 375 ++++++++++++++++++++
3 files changed, 393 insertions(+)
diff --git a/crypto/Kconfig b/crypto/Kconfig
index e8269d1b0282..5a51b877277e 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -896,6 +896,23 @@ config CRYPTO_AES
See <http://csrc.nist.gov/CryptoToolkit/aes/> for more information.
+config CRYPTO_AES_TI
+ tristate "Fixed time AES cipher"
+ select CRYPTO_ALGAPI
+ help
+ This is a generic implementation of AES that attempts to eliminate
+ data dependent latencies as much as possible without affecting
+ performance too much. It is intended for use by the generic CCM
+ and GCM drivers, and other CTR or CMAC/XCBC based modes that rely
+ solely on encryption (although decryption is supported as well, but
+ with a more dramatic performance hit)
+
+ Instead of using 16 lookup tables of 1 KB each, (8 for encryption and
+ 8 for decryption), this implementation only uses just two S-boxes of
+ 256 bytes each, and attempts to eliminate data dependent latencies by
+ prefetching the entire table into the cache at the start of each
+ block.
+
config CRYPTO_AES_586
tristate "AES cipher algorithms (i586)"
depends on (X86 || UML_X86) && !64BIT
diff --git a/crypto/Makefile b/crypto/Makefile
index b8f0e3eb0791..bcd834536163 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -99,6 +99,7 @@ obj-$(CONFIG_CRYPTO_TWOFISH) += twofish_generic.o
obj-$(CONFIG_CRYPTO_TWOFISH_COMMON) += twofish_common.o
obj-$(CONFIG_CRYPTO_SERPENT) += serpent_generic.o
obj-$(CONFIG_CRYPTO_AES) += aes_generic.o
+obj-$(CONFIG_CRYPTO_AES_TI) += aes_ti.o
obj-$(CONFIG_CRYPTO_CAMELLIA) += camellia_generic.o
obj-$(CONFIG_CRYPTO_CAST_COMMON) += cast_common.o
obj-$(CONFIG_CRYPTO_CAST5) += cast5_generic.o
diff --git a/crypto/aes_ti.c b/crypto/aes_ti.c
new file mode 100644
index 000000000000..92644fd1ac19
--- /dev/null
+++ b/crypto/aes_ti.c
@@ -0,0 +1,375 @@
+/*
+ * Scalar fixed time AES core transform
+ *
+ * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <crypto/aes.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+#include <asm/unaligned.h>
+
+/*
+ * Emit the sbox as volatile const to prevent the compiler from doing
+ * constant folding on sbox references involving fixed indexes.
+ */
+static volatile const u8 __cacheline_aligned __aesti_sbox[] = {
+ 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5,
+ 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
+ 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0,
+ 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
+ 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc,
+ 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
+ 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a,
+ 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
+ 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0,
+ 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
+ 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b,
+ 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
+ 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85,
+ 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
+ 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5,
+ 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
+ 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17,
+ 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
+ 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88,
+ 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
+ 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c,
+ 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
+ 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9,
+ 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
+ 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6,
+ 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
+ 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e,
+ 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
+ 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94,
+ 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
+ 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68,
+ 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16,
+};
+
+static volatile const u8 __cacheline_aligned __aesti_inv_sbox[] = {
+ 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38,
+ 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb,
+ 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87,
+ 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb,
+ 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d,
+ 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
+ 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2,
+ 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25,
+ 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16,
+ 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92,
+ 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda,
+ 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
+ 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a,
+ 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06,
+ 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02,
+ 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b,
+ 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea,
+ 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
+ 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85,
+ 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e,
+ 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89,
+ 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b,
+ 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20,
+ 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
+ 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31,
+ 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f,
+ 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d,
+ 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef,
+ 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0,
+ 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
+ 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26,
+ 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d,
+};
+
+static u32 mul_by_x(u32 w)
+{
+ u32 x = w & 0x7f7f7f7f;
+ u32 y = w & 0x80808080;
+
+ /* multiply by polynomial 'x' (0b10) in GF(2^8) */
+ return (x << 1) ^ (y >> 7) * 0x1b;
+}
+
+static u32 mul_by_x2(u32 w)
+{
+ u32 x = w & 0x3f3f3f3f;
+ u32 y = w & 0x80808080;
+ u32 z = w & 0x40404040;
+
+ /* multiply by polynomial 'x^2' (0b100) in GF(2^8) */
+ return (x << 2) ^ (y >> 7) * 0x36 ^ (z >> 6) * 0x1b;
+}
+
+static u32 mix_columns(u32 x)
+{
+ /*
+ * Perform the following matrix multiplication in GF(2^8)
+ *
+ * | 0x2 0x3 0x1 0x1 | | x[0] |
+ * | 0x1 0x2 0x3 0x1 | | x[1] |
+ * | 0x1 0x1 0x2 0x3 | x | x[2] |
+ * | 0x3 0x1 0x1 0x3 | | x[3] |
+ */
+ u32 y = mul_by_x(x) ^ ror32(x, 16);
+
+ return y ^ ror32(x ^ y, 8);
+}
+
+static u32 inv_mix_columns(u32 x)
+{
+ /*
+ * Perform the following matrix multiplication in GF(2^8)
+ *
+ * | 0xe 0xb 0xd 0x9 | | x[0] |
+ * | 0x9 0xe 0xb 0xd | | x[1] |
+ * | 0xd 0x9 0xe 0xb | x | x[2] |
+ * | 0xb 0xd 0x9 0xe | | x[3] |
+ *
+ * which can conveniently be reduced to
+ *
+ * | 0x2 0x3 0x1 0x1 | | 0x5 0x0 0x4 0x0 | | x[0] |
+ * | 0x1 0x2 0x3 0x1 | | 0x0 0x5 0x0 0x4 | | x[1] |
+ * | 0x1 0x1 0x2 0x3 | x | 0x4 0x0 0x5 0x0 | x | x[2] |
+ * | 0x3 0x1 0x1 0x2 | | 0x0 0x4 0x0 0x5 | | x[3] |
+ */
+ u32 y = mul_by_x2(x);
+
+ return mix_columns(x ^ y ^ ror32(y, 16));
+}
+
+static __always_inline u32 subshift(u32 in[], int pos)
+{
+ return (__aesti_sbox[in[pos] & 0xff]) ^
+ (__aesti_sbox[(in[(pos + 1) % 4] >> 8) & 0xff] << 8) ^
+ (__aesti_sbox[(in[(pos + 2) % 4] >> 16) & 0xff] << 16) ^
+ (__aesti_sbox[(in[(pos + 3) % 4] >> 24) & 0xff] << 24);
+}
+
+static __always_inline u32 inv_subshift(u32 in[], int pos)
+{
+ return (__aesti_inv_sbox[in[pos] & 0xff]) ^
+ (__aesti_inv_sbox[(in[(pos + 3) % 4] >> 8) & 0xff] << 8) ^
+ (__aesti_inv_sbox[(in[(pos + 2) % 4] >> 16) & 0xff] << 16) ^
+ (__aesti_inv_sbox[(in[(pos + 1) % 4] >> 24) & 0xff] << 24);
+}
+
+static u32 subw(u32 in)
+{
+ return (__aesti_sbox[in & 0xff]) ^
+ (__aesti_sbox[(in >> 8) & 0xff] << 8) ^
+ (__aesti_sbox[(in >> 16) & 0xff] << 16) ^
+ (__aesti_sbox[(in >> 24) & 0xff] << 24);
+}
+
+static int aesti_expand_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
+ unsigned int key_len)
+{
+ u32 kwords = key_len / sizeof(u32);
+ u32 rc, i, j;
+
+ if (key_len != AES_KEYSIZE_128 &&
+ key_len != AES_KEYSIZE_192 &&
+ key_len != AES_KEYSIZE_256)
+ return -EINVAL;
+
+ ctx->key_length = key_len;
+
+ for (i = 0; i < kwords; i++)
+ ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));
+
+ for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) {
+ u32 *rki = ctx->key_enc + (i * kwords);
+ u32 *rko = rki + kwords;
+
+ rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0];
+ rko[1] = rko[0] ^ rki[1];
+ rko[2] = rko[1] ^ rki[2];
+ rko[3] = rko[2] ^ rki[3];
+
+ if (key_len == 24) {
+ if (i >= 7)
+ break;
+ rko[4] = rko[3] ^ rki[4];
+ rko[5] = rko[4] ^ rki[5];
+ } else if (key_len == 32) {
+ if (i >= 6)
+ break;
+ rko[4] = subw(rko[3]) ^ rki[4];
+ rko[5] = rko[4] ^ rki[5];
+ rko[6] = rko[5] ^ rki[6];
+ rko[7] = rko[6] ^ rki[7];
+ }
+ }
+
+ /*
+ * Generate the decryption keys for the Equivalent Inverse Cipher.
+ * This involves reversing the order of the round keys, and applying
+ * the Inverse Mix Columns transformation to all but the first and
+ * the last one.
+ */
+ ctx->key_dec[0] = ctx->key_enc[key_len + 24];
+ ctx->key_dec[1] = ctx->key_enc[key_len + 25];
+ ctx->key_dec[2] = ctx->key_enc[key_len + 26];
+ ctx->key_dec[3] = ctx->key_enc[key_len + 27];
+
+ for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) {
+ ctx->key_dec[i] = inv_mix_columns(ctx->key_enc[j]);
+ ctx->key_dec[i + 1] = inv_mix_columns(ctx->key_enc[j + 1]);
+ ctx->key_dec[i + 2] = inv_mix_columns(ctx->key_enc[j + 2]);
+ ctx->key_dec[i + 3] = inv_mix_columns(ctx->key_enc[j + 3]);
+ }
+
+ ctx->key_dec[i] = ctx->key_enc[0];
+ ctx->key_dec[i + 1] = ctx->key_enc[1];
+ ctx->key_dec[i + 2] = ctx->key_enc[2];
+ ctx->key_dec[i + 3] = ctx->key_enc[3];
+
+ return 0;
+}
+
+static int aesti_set_key(struct crypto_tfm *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ int err;
+
+ err = aesti_expand_key(ctx, in_key, key_len);
+ if (err)
+ return err;
+
+ /*
+ * In order to force the compiler to emit data independent Sbox lookups
+ * at the start of each block, xor the first round key with values at
+ * fixed indexes in the Sbox. This will need to be repeated each time
+ * the key is used, which will pull the entire Sbox into the D-cache
+ * before any data dependent Sbox lookups are performed.
+ */
+ ctx->key_enc[0] ^= __aesti_sbox[ 0] ^ __aesti_sbox[128];
+ ctx->key_enc[1] ^= __aesti_sbox[32] ^ __aesti_sbox[160];
+ ctx->key_enc[2] ^= __aesti_sbox[64] ^ __aesti_sbox[192];
+ ctx->key_enc[3] ^= __aesti_sbox[96] ^ __aesti_sbox[224];
+
+ ctx->key_dec[0] ^= __aesti_inv_sbox[ 0] ^ __aesti_inv_sbox[128];
+ ctx->key_dec[1] ^= __aesti_inv_sbox[32] ^ __aesti_inv_sbox[160];
+ ctx->key_dec[2] ^= __aesti_inv_sbox[64] ^ __aesti_inv_sbox[192];
+ ctx->key_dec[3] ^= __aesti_inv_sbox[96] ^ __aesti_inv_sbox[224];
+
+ return 0;
+}
+
+static void aesti_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ const u32 *rkp = ctx->key_enc + 4;
+ int rounds = 6 + ctx->key_length / 4;
+ u32 st0[4], st1[4];
+ int round;
+
+ st0[0] = ctx->key_enc[0] ^ get_unaligned_le32(in);
+ st0[1] = ctx->key_enc[1] ^ get_unaligned_le32(in + 4);
+ st0[2] = ctx->key_enc[2] ^ get_unaligned_le32(in + 8);
+ st0[3] = ctx->key_enc[3] ^ get_unaligned_le32(in + 12);
+
+ st0[0] ^= __aesti_sbox[ 0] ^ __aesti_sbox[128];
+ st0[1] ^= __aesti_sbox[32] ^ __aesti_sbox[160];
+ st0[2] ^= __aesti_sbox[64] ^ __aesti_sbox[192];
+ st0[3] ^= __aesti_sbox[96] ^ __aesti_sbox[224];
+
+ for (round = 0;; round += 2, rkp += 8) {
+ st1[0] = mix_columns(subshift(st0, 0)) ^ rkp[0];
+ st1[1] = mix_columns(subshift(st0, 1)) ^ rkp[1];
+ st1[2] = mix_columns(subshift(st0, 2)) ^ rkp[2];
+ st1[3] = mix_columns(subshift(st0, 3)) ^ rkp[3];
+
+ if (round == rounds - 2)
+ break;
+
+ st0[0] = mix_columns(subshift(st1, 0)) ^ rkp[4];
+ st0[1] = mix_columns(subshift(st1, 1)) ^ rkp[5];
+ st0[2] = mix_columns(subshift(st1, 2)) ^ rkp[6];
+ st0[3] = mix_columns(subshift(st1, 3)) ^ rkp[7];
+ }
+
+ put_unaligned_le32(subshift(st1, 0) ^ rkp[4], out);
+ put_unaligned_le32(subshift(st1, 1) ^ rkp[5], out + 4);
+ put_unaligned_le32(subshift(st1, 2) ^ rkp[6], out + 8);
+ put_unaligned_le32(subshift(st1, 3) ^ rkp[7], out + 12);
+}
+
+static void aesti_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ const u32 *rkp = ctx->key_dec + 4;
+ int rounds = 6 + ctx->key_length / 4;
+ u32 st0[4], st1[4];
+ int round;
+
+ st0[0] = ctx->key_dec[0] ^ get_unaligned_le32(in);
+ st0[1] = ctx->key_dec[1] ^ get_unaligned_le32(in + 4);
+ st0[2] = ctx->key_dec[2] ^ get_unaligned_le32(in + 8);
+ st0[3] = ctx->key_dec[3] ^ get_unaligned_le32(in + 12);
+
+ st0[0] ^= __aesti_inv_sbox[ 0] ^ __aesti_inv_sbox[128];
+ st0[1] ^= __aesti_inv_sbox[32] ^ __aesti_inv_sbox[160];
+ st0[2] ^= __aesti_inv_sbox[64] ^ __aesti_inv_sbox[192];
+ st0[3] ^= __aesti_inv_sbox[96] ^ __aesti_inv_sbox[224];
+
+ for (round = 0;; round += 2, rkp += 8) {
+ st1[0] = inv_mix_columns(inv_subshift(st0, 0)) ^ rkp[0];
+ st1[1] = inv_mix_columns(inv_subshift(st0, 1)) ^ rkp[1];
+ st1[2] = inv_mix_columns(inv_subshift(st0, 2)) ^ rkp[2];
+ st1[3] = inv_mix_columns(inv_subshift(st0, 3)) ^ rkp[3];
+
+ if (round == rounds - 2)
+ break;
+
+ st0[0] = inv_mix_columns(inv_subshift(st1, 0)) ^ rkp[4];
+ st0[1] = inv_mix_columns(inv_subshift(st1, 1)) ^ rkp[5];
+ st0[2] = inv_mix_columns(inv_subshift(st1, 2)) ^ rkp[6];
+ st0[3] = inv_mix_columns(inv_subshift(st1, 3)) ^ rkp[7];
+ }
+
+ put_unaligned_le32(inv_subshift(st1, 0) ^ rkp[4], out);
+ put_unaligned_le32(inv_subshift(st1, 1) ^ rkp[5], out + 4);
+ put_unaligned_le32(inv_subshift(st1, 2) ^ rkp[6], out + 8);
+ put_unaligned_le32(inv_subshift(st1, 3) ^ rkp[7], out + 12);
+}
+
+static struct crypto_alg aes_alg = {
+ .cra_name = "aes",
+ .cra_driver_name = "aes-fixed-time",
+ .cra_priority = 100 + 1,
+ .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct crypto_aes_ctx),
+ .cra_module = THIS_MODULE,
+
+ .cra_cipher.cia_min_keysize = AES_MIN_KEY_SIZE,
+ .cra_cipher.cia_max_keysize = AES_MAX_KEY_SIZE,
+ .cra_cipher.cia_setkey = aesti_set_key,
+ .cra_cipher.cia_encrypt = aesti_encrypt,
+ .cra_cipher.cia_decrypt = aesti_decrypt
+};
+
+static int __init aes_init(void)
+{
+ return crypto_register_alg(&aes_alg);
+}
+
+static void __exit aes_fini(void)
+{
+ crypto_unregister_alg(&aes_alg);
+}
+
+module_init(aes_init);
+module_exit(aes_fini);
+
+MODULE_DESCRIPTION("Generic fixed time AES");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
--
2.7.4
^ permalink raw reply related
* Re: crypto: NULL deref in sha512_mb_mgr_get_comp_job_avx2
From: Tim Chen @ 2017-02-02 16:00 UTC (permalink / raw)
To: Dmitry Vyukov
Cc: Herbert Xu, David Miller, linux-crypto, LKML, megha.dey,
fenghua.yu, syzkaller
In-Reply-To: <CACT4Y+bwD4ktMKVEkcmvJ_EuW834jAZbJfANxfyDKpmGXFxr4Q@mail.gmail.com>
On Thu, 2017-02-02 at 11:58 +0100, Dmitry Vyukov wrote:
> On Wed, Feb 1, 2017 at 7:45 PM, Tim Chen <tim.c.chen@linux.intel.com> wrote:
> >
> > On Tue, Jan 31, 2017 at 02:16:31PM +0100, Dmitry Vyukov wrote:
> > >
> > > Hello,
> > >
> > > I am getting the following reports with low frequency while running
> > > syzkaller fuzzer. Unfortunately they are not reproducible and happen
> > > in a background thread, so it is difficult to extract any context on
> > > my side. I see only few such crashes per week, so most likely it is
> > > some hard to trigger data race. The following reports are from mmotm
> > > tree, commits 00e20cfc2bf04a0cbe1f5405f61c8426f43eee84 and
> > > fff7e71eac7788904753136f09bcad7471f7799e. Any ideas as to how this can
> > > happen?
> > >
> > > BUG: unable to handle kernel NULL pointer dereference at 0000000000000060
> > > IP: [<ffffffff813fc09e>] sha512_mb_mgr_get_comp_job_avx2+0x6e/0xee
> > > arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S:251
> > > PGD 1d2395067 [ 220.874864] PUD 1d2860067
> > > Oops: 0002 [#1] SMP KASAN
> > > Dumping ftrace buffer:
> > > (ftrace buffer empty)
> > > Modules linked in:
> > > CPU: 0 PID: 516 Comm: kworker/0:1 Not tainted 4.9.0 #4
> > > Hardware name: Google Google Compute Engine/Google Compute Engine,
> > > BIOS Google 01/01/2011
> > > Workqueue: crypto mcryptd_queue_worker
> > > task: ffff8801d9f346c0 task.stack: ffff8801d9f08000
> > > RIP: 0010:[<ffffffff813fc09e>] [<ffffffff813fc09e>]
> > > sha512_mb_mgr_get_comp_job_avx2+0x6e/0xee
> > > arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S:251
> > > RSP: 0018:ffff8801d9f0eef8 EFLAGS: 00010202
> > > RAX: 0000000000000000 RBX: ffff8801d7db1190 RCX: 0000000000000006
> > > RDX: 0000000000000001 RSI: ffff8801d9f34ee8 RDI: ffff8801d7db1040
> > > RBP: ffff8801d9f0f258 R08: 0000000100000000 R09: 0000000000000001
> > > R10: 0000000000000002 R11: 0000000000000003 R12: ffff8801d9f0f230
> > > R13: ffff8801c8bbc4e0 R14: ffff8801c8bbc530 R15: ffff8801d9f0ef70
> > > FS: 0000000000000000(0000) GS:ffff8801dc000000(0000) knlGS:0000000000000000
> > > CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
> > > CR2: 0000000000000060 CR3: 00000001cc15a000 CR4: 00000000001406f0
> > > DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
> > > DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
> > > Stack:
> > > ffff8801d7db1040 ffffffff813fa207 dffffc0000000000 ffffe8ffffc0f238
> > > 0000000000000002 1ffff1003b3e1dea ffffe8ffffc0f218 ffff8801d9f0f190
> > > 0000000000000282 ffffe8ffffc0f140 ffffe8ffffc0f220 0000000041b58ab3
> > > Call Trace:
> > > [<ffffffff813fb407>] sha512_mb_update+0x2f7/0x4e0
> > > arch/x86/crypto/sha512-mb/sha512_mb.c:588
> > > [<ffffffff8219d4ad>] crypto_ahash_update include/crypto/hash.h:512 [inline]
> > > [<ffffffff8219d4ad>] ahash_mcryptd_update crypto/mcryptd.c:627 [inline]
> > > [<ffffffff8219d4ad>] mcryptd_hash_update+0xcd/0x1c0 crypto/mcryptd.c:373
> > > [<ffffffff8219c99f>] mcryptd_queue_worker+0xff/0x6a0 crypto/mcryptd.c:181
> > > [<ffffffff81492960>] process_one_work+0xbd0/0x1c10 kernel/workqueue.c:2096
> > > [<ffffffff81493bc3>] worker_thread+0x223/0x1990 kernel/workqueue.c:2230
> > > [<ffffffff814abb33>] kthread+0x323/0x3e0 kernel/kthread.c:209
> > > [<ffffffff8436fbaa>] ret_from_fork+0x2a/0x40 arch/x86/entry/entry_64.S:433
> > > Code: 49 0f 42 d3 48 f7 c2 f0 ff ff ff 0f 85 9a 00 00 00 48 83 e2 0f
> > > 48 6b da 08 48 8d 9c 1f 48 01 00 00 48 8b 03 48 c7 03 00 00 00 00 <c7>
> > > 40 60 02 00 00 00 48 8b 9f 40 01 00 00 48 c1 e3 08 48 09 d3
> > > RIP [<ffffffff813fc09e>] sha512_mb_mgr_get_comp_job_avx2+0x6e/0xee
> > > arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S:251
> > > RSP <ffff8801d9f0eef8>
> > > CR2: 0000000000000060
> > > ---[ end trace 139fd4cda5dfe2c4 ]---
> > >
> > Dmitry,
> >
> > One theory that Mehga and I have is that perhaps the flusher
> > and regular computaion updates are stepping on each other.
> > Can you try this patch and see if it helps?
>
> No, for this one I can't. Sorry.
> It happens with very low frequency and only one fuzzer that tests
> mmotm tree. If/when this is committed, I can keep an eye on these
> reports and notify if I still see them.
> If you have a hypothesis as to how it happens, perhaps you could write
> a test that provokes the crash and maybe add some sleeps to kernel
> code or alter timeouts to increase probability.
>
If this patch is merged, it will most likely go into Herbert's crypto-dev
tree and not Andrew's mm tree for testing. We will try to do the best
on our side for testing to replicate the crash scenario.
Will it be possible to have one of your fuzzer run the crypto-dev tree
once the patch got merged there?
Thanks.
Tim
>
>
> >
> > --->8---
> >
> > From: Tim Chen <tim.c.chen@linux.intel.com>
> > Subject: [PATCH] crypto/sha512-mb: Protect sha512 mb ctx mgr access
> > To: Herbert Xu <herbert@gondor.apana.org.au>, Dmitry Vyukov <dvyukov@google.com>
> > Cc: Tim Chen <tim.c.chen@linux.intel.com>, David Miller <davem@davemloft.net>, linux-crypto@vger.kernel.org, LKML <linux-kernel@vger.kernel.org>, megha.dey@linux.intel.com, fenghua.yu@intel.com
> >
> > The flusher and regular multi-buffer computation via mcryptd may race with another.
> > Add here a lock and turn off interrupt to to access multi-buffer
> > computation state cstate->mgr before a round of computation. This should
> > prevent the flusher code jumping in.
> >
> > Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
> > ---
> > arch/x86/crypto/sha512-mb/sha512_mb.c | 64 +++++++++++++++++++++++------------
> > 1 file changed, 42 insertions(+), 22 deletions(-)
> >
> > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb.c b/arch/x86/crypto/sha512-mb/sha512_mb.c
> > index d210174..f3c1c21 100644
> > --- a/arch/x86/crypto/sha512-mb/sha512_mb.c
> > +++ b/arch/x86/crypto/sha512-mb/sha512_mb.c
> > @@ -221,7 +221,7 @@ static struct sha512_hash_ctx *sha512_ctx_mgr_resubmit
> > }
> >
> > static struct sha512_hash_ctx
> > - *sha512_ctx_mgr_get_comp_ctx(struct sha512_ctx_mgr *mgr)
> > + *sha512_ctx_mgr_get_comp_ctx(struct mcryptd_alg_cstate *cstate)
> > {
> > /*
> > * If get_comp_job returns NULL, there are no jobs complete.
> > @@ -233,11 +233,17 @@ static struct sha512_hash_ctx
> > * Otherwise, all jobs currently being managed by the hash_ctx_mgr
> > * still need processing.
> > */
> > + struct sha512_ctx_mgr *mgr;
> > struct sha512_hash_ctx *ctx;
> > + unsigned long flags;
> >
> > + mgr = cstate->mgr;
> > + spin_lock_irqsave(&cstate->work_lock, flags);
> > ctx = (struct sha512_hash_ctx *)
> > sha512_job_mgr_get_comp_job(&mgr->mgr);
> > - return sha512_ctx_mgr_resubmit(mgr, ctx);
> > + ctx = sha512_ctx_mgr_resubmit(mgr, ctx);
> > + spin_unlock_irqrestore(&cstate->work_lock, flags);
> > + return ctx;
> > }
> >
> > static void sha512_ctx_mgr_init(struct sha512_ctx_mgr *mgr)
> > @@ -246,12 +252,17 @@ static void sha512_ctx_mgr_init(struct sha512_ctx_mgr *mgr)
> > }
> >
> > static struct sha512_hash_ctx
> > - *sha512_ctx_mgr_submit(struct sha512_ctx_mgr *mgr,
> > + *sha512_ctx_mgr_submit(struct mcryptd_alg_cstate *cstate,
> > struct sha512_hash_ctx *ctx,
> > const void *buffer,
> > uint32_t len,
> > int flags)
> > {
> > + struct sha512_ctx_mgr *mgr;
> > + unsigned long irqflags;
> > +
> > + mgr = cstate->mgr;
> > + spin_lock_irqsave(&cstate->work_lock, irqflags);
> > if (flags & (~HASH_ENTIRE)) {
> > /*
> > * User should not pass anything other than FIRST, UPDATE, or
> > @@ -351,20 +362,26 @@ static struct sha512_hash_ctx
> > }
> > }
> >
> > - return sha512_ctx_mgr_resubmit(mgr, ctx);
> > + ctx = sha512_ctx_mgr_resubmit(mgr, ctx);
> > + spin_unlock_irqrestore(&cstate->work_lock, irqflags);
> > + return ctx;
> > }
> >
> > -static struct sha512_hash_ctx *sha512_ctx_mgr_flush(struct sha512_ctx_mgr *mgr)
> > +static struct sha512_hash_ctx *sha512_ctx_mgr_flush(struct mcryptd_alg_cstate *cstate)
> > {
> > + struct sha512_ctx_mgr *mgr;
> > struct sha512_hash_ctx *ctx;
> > + unsigned long flags;
> >
> > + mgr = cstate->mgr;
> > + spin_lock_irqsave(&cstate->work_lock, flags);
> > while (1) {
> > ctx = (struct sha512_hash_ctx *)
> > sha512_job_mgr_flush(&mgr->mgr);
> >
> > /* If flush returned 0, there are no more jobs in flight. */
> > if (!ctx)
> > - return NULL;
> > + break;
> >
> > /*
> > * If flush returned a job, resubmit the job to finish
> > @@ -378,8 +395,10 @@ static struct sha512_hash_ctx *sha512_ctx_mgr_flush(struct sha512_ctx_mgr *mgr)
> > * the sha512_ctx_mgr still need processing. Loop.
> > */
> > if (ctx)
> > - return ctx;
> > + break;
> > }
> > + spin_unlock_irqrestore(&cstate->work_lock, flags);
> > + return ctx;
> > }
> >
> > static int sha512_mb_init(struct ahash_request *areq)
> > @@ -439,11 +458,11 @@ static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx,
> > sha_ctx = (struct sha512_hash_ctx *)
> > ahash_request_ctx(&rctx->areq);
> > kernel_fpu_begin();
> > - sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx,
> > + sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx,
> > rctx->walk.data, nbytes, flag);
> > if (!sha_ctx) {
> > if (flush)
> > - sha_ctx = sha512_ctx_mgr_flush(cstate->mgr);
> > + sha_ctx = sha512_ctx_mgr_flush(cstate);
> > }
> > kernel_fpu_end();
> > if (sha_ctx)
> > @@ -471,11 +490,12 @@ static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
> > struct sha512_hash_ctx *sha_ctx;
> > struct mcryptd_hash_request_ctx *req_ctx;
> > int ret;
> > + unsigned long flags;
> >
> > /* remove from work list */
> > - spin_lock(&cstate->work_lock);
> > + spin_lock_irqsave(&cstate->work_lock, flags);
> > list_del(&rctx->waiter);
> > - spin_unlock(&cstate->work_lock);
> > + spin_unlock_irqrestore(&cstate->work_lock, flags);
> >
> > if (irqs_disabled())
> > rctx->complete(&req->base, err);
> > @@ -486,14 +506,14 @@ static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
> > }
> >
> > /* check to see if there are other jobs that are done */
> > - sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate->mgr);
> > + sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate);
> > while (sha_ctx) {
> > req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx);
> > ret = sha_finish_walk(&req_ctx, cstate, false);
> > if (req_ctx) {
> > - spin_lock(&cstate->work_lock);
> > + spin_lock_irqsave(&cstate->work_lock, flags);
> > list_del(&req_ctx->waiter);
> > - spin_unlock(&cstate->work_lock);
> > + spin_unlock_irqrestore(&cstate->work_lock, flags);
> >
> > req = cast_mcryptd_ctx_to_req(req_ctx);
> > if (irqs_disabled())
> > @@ -504,7 +524,7 @@ static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
> > local_bh_enable();
> > }
> > }
> > - sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate->mgr);
> > + sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate);
> > }
> >
> > return 0;
> > @@ -515,6 +535,7 @@ static void sha512_mb_add_list(struct mcryptd_hash_request_ctx *rctx,
> > {
> > unsigned long next_flush;
> > unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL);
> > + unsigned long flags;
> >
> > /* initialize tag */
> > rctx->tag.arrival = jiffies; /* tag the arrival time */
> > @@ -522,9 +543,9 @@ static void sha512_mb_add_list(struct mcryptd_hash_request_ctx *rctx,
> > next_flush = rctx->tag.arrival + delay;
> > rctx->tag.expire = next_flush;
> >
> > - spin_lock(&cstate->work_lock);
> > + spin_lock_irqsave(&cstate->work_lock, flags);
> > list_add_tail(&rctx->waiter, &cstate->work_list);
> > - spin_unlock(&cstate->work_lock);
> > + spin_unlock_irqrestore(&cstate->work_lock, flags);
> >
> > mcryptd_arm_flusher(cstate, delay);
> > }
> > @@ -565,7 +586,7 @@ static int sha512_mb_update(struct ahash_request *areq)
> > sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq);
> > sha512_mb_add_list(rctx, cstate);
> > kernel_fpu_begin();
> > - sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
> > + sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, rctx->walk.data,
> > nbytes, HASH_UPDATE);
> > kernel_fpu_end();
> >
> > @@ -628,7 +649,7 @@ static int sha512_mb_finup(struct ahash_request *areq)
> > sha512_mb_add_list(rctx, cstate);
> >
> > kernel_fpu_begin();
> > - sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
> > + sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, rctx->walk.data,
> > nbytes, flag);
> > kernel_fpu_end();
> >
> > @@ -677,8 +698,7 @@ static int sha512_mb_final(struct ahash_request *areq)
> > /* flag HASH_FINAL and 0 data size */
> > sha512_mb_add_list(rctx, cstate);
> > kernel_fpu_begin();
> > - sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0,
> > - HASH_LAST);
> > + sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, &data, 0, HASH_LAST);
> > kernel_fpu_end();
> >
> > /* check if anything is returned */
> > @@ -940,7 +960,7 @@ static unsigned long sha512_mb_flusher(struct mcryptd_alg_cstate *cstate)
> > break;
> > kernel_fpu_begin();
> > sha_ctx = (struct sha512_hash_ctx *)
> > - sha512_ctx_mgr_flush(cstate->mgr);
> > + sha512_ctx_mgr_flush(cstate);
> > kernel_fpu_end();
> > if (!sha_ctx) {
> > pr_err("sha512_mb error: nothing got flushed for"
> > --
> > 2.5.5
> >
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox