* [PATCH v3 02/12] crypto: atmel-sha: update request queue management to make it more generic
From: Cyrille Pitchen @ 2017-01-26 16:07 UTC (permalink / raw)
To: herbert, davem, nicolas.ferre
Cc: smueller, linux-crypto, linux-kernel, linux-arm-kernel,
Cyrille Pitchen
In-Reply-To: <cover.1485443478.git.cyrille.pitchen@atmel.com>
This patch is a transitional patch. It splits the atmel_sha_handle_queue()
function. Now atmel_sha_handle_queue() only manages the request queue and
calls a new .start() hook from the atmel_sha_ctx structure.
This hook allows to implement different kind of requests still handled by
a single queue.
Also when the req parameter of atmel_sha_handle_queue() refers to the very
same request as the one returned by crypto_dequeue_request(), the queue
management now gives a chance to this crypto request to be handled
synchronously, hence reducing latencies. The .start() hook returns 0 if
the crypto request was handled synchronously and -EINPROGRESS if the
crypto request still need to be handled asynchronously.
Besides, the new .is_async member of the atmel_sha_dev structure helps
tagging this asynchronous state. Indeed, the req->base.complete() callback
should not be called if the crypto request is handled synchronously.
Signed-off-by: Cyrille Pitchen <cyrille.pitchen@atmel.com>
---
drivers/crypto/atmel-sha.c | 74 +++++++++++++++++++++++++++++++++-------------
1 file changed, 54 insertions(+), 20 deletions(-)
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index 33a36e667547..2dbed8bb8d26 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -105,8 +105,11 @@ struct atmel_sha_reqctx {
u8 buffer[SHA_BUFFER_LEN + SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
};
+typedef int (*atmel_sha_fn_t)(struct atmel_sha_dev *);
+
struct atmel_sha_ctx {
struct atmel_sha_dev *dd;
+ atmel_sha_fn_t start;
unsigned long flags;
};
@@ -134,6 +137,7 @@ struct atmel_sha_dev {
unsigned long flags;
struct crypto_queue queue;
struct ahash_request *req;
+ bool is_async;
struct atmel_sha_dma dma_lch_in;
@@ -163,6 +167,24 @@ static inline void atmel_sha_write(struct atmel_sha_dev *dd,
writel_relaxed(value, dd->io_base + offset);
}
+static inline int atmel_sha_complete(struct atmel_sha_dev *dd, int err)
+{
+ struct ahash_request *req = dd->req;
+
+ dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU |
+ SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY);
+
+ clk_disable(dd->iclk);
+
+ if (dd->is_async && req->base.complete)
+ req->base.complete(&req->base, err);
+
+ /* handle new request */
+ tasklet_schedule(&dd->queue_task);
+
+ return err;
+}
+
static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx)
{
size_t count;
@@ -474,6 +496,8 @@ static void atmel_sha_dma_callback(void *data)
{
struct atmel_sha_dev *dd = data;
+ dd->is_async = true;
+
/* dma_lch_in - completed - wait DATRDY */
atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);
}
@@ -509,7 +533,7 @@ static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
}
if (!in_desc)
- return -EINVAL;
+ atmel_sha_complete(dd, -EINVAL);
in_desc->callback = atmel_sha_dma_callback;
in_desc->callback_param = dd;
@@ -566,7 +590,7 @@ static int atmel_sha_xmit_dma_map(struct atmel_sha_dev *dd,
if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen +
ctx->block_size);
- return -EINVAL;
+ atmel_sha_complete(dd, -EINVAL);
}
ctx->flags &= ~SHA_FLAGS_SG;
@@ -657,7 +681,7 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd)
if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
dev_err(dd->dev, "dma %u bytes error\n",
ctx->buflen + ctx->block_size);
- return -EINVAL;
+ atmel_sha_complete(dd, -EINVAL);
}
if (length == 0) {
@@ -671,7 +695,7 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd)
if (!dma_map_sg(dd->dev, ctx->sg, 1,
DMA_TO_DEVICE)) {
dev_err(dd->dev, "dma_map_sg error\n");
- return -EINVAL;
+ atmel_sha_complete(dd, -EINVAL);
}
ctx->flags |= SHA_FLAGS_SG;
@@ -685,7 +709,7 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd)
if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
dev_err(dd->dev, "dma_map_sg error\n");
- return -EINVAL;
+ atmel_sha_complete(dd, -EINVAL);
}
ctx->flags |= SHA_FLAGS_SG;
@@ -843,16 +867,7 @@ static void atmel_sha_finish_req(struct ahash_request *req, int err)
}
/* atomic operation is not needed here */
- dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU |
- SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY);
-
- clk_disable(dd->iclk);
-
- if (req->base.complete)
- req->base.complete(&req->base, err);
-
- /* handle new request */
- tasklet_schedule(&dd->queue_task);
+ (void)atmel_sha_complete(dd, err);
}
static int atmel_sha_hw_init(struct atmel_sha_dev *dd)
@@ -893,8 +908,9 @@ static int atmel_sha_handle_queue(struct atmel_sha_dev *dd,
struct ahash_request *req)
{
struct crypto_async_request *async_req, *backlog;
- struct atmel_sha_reqctx *ctx;
+ struct atmel_sha_ctx *ctx;
unsigned long flags;
+ bool start_async;
int err = 0, ret = 0;
spin_lock_irqsave(&dd->lock, flags);
@@ -919,9 +935,22 @@ static int atmel_sha_handle_queue(struct atmel_sha_dev *dd,
if (backlog)
backlog->complete(backlog, -EINPROGRESS);
- req = ahash_request_cast(async_req);
- dd->req = req;
- ctx = ahash_request_ctx(req);
+ ctx = crypto_tfm_ctx(async_req->tfm);
+
+ dd->req = ahash_request_cast(async_req);
+ start_async = (dd->req != req);
+ dd->is_async = start_async;
+
+ /* WARNING: ctx->start() MAY change dd->is_async. */
+ err = ctx->start(dd);
+ return (start_async) ? ret : err;
+}
+
+static int atmel_sha_start(struct atmel_sha_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
+ int err;
dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
ctx->op, req->nbytes);
@@ -947,7 +976,7 @@ static int atmel_sha_handle_queue(struct atmel_sha_dev *dd,
dev_dbg(dd->dev, "exit, err: %d\n", err);
- return ret;
+ return err;
}
static int atmel_sha_enqueue(struct ahash_request *req, unsigned int op)
@@ -1043,8 +1072,11 @@ static int atmel_sha_import(struct ahash_request *req, const void *in)
static int atmel_sha_cra_init(struct crypto_tfm *tfm)
{
+ struct atmel_sha_ctx *ctx = crypto_tfm_ctx(tfm);
+
crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
sizeof(struct atmel_sha_reqctx));
+ ctx->start = atmel_sha_start;
return 0;
}
@@ -1188,6 +1220,8 @@ static void atmel_sha_done_task(unsigned long data)
struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data;
int err = 0;
+ dd->is_async = true;
+
if (SHA_FLAGS_CPU & dd->flags) {
if (SHA_FLAGS_OUTPUT_READY & dd->flags) {
dd->flags &= ~SHA_FLAGS_OUTPUT_READY;
--
2.7.4
^ permalink raw reply related
* [PATCH v3 03/12] crypto: atmel-sha: make atmel_sha_done_task more generic
From: Cyrille Pitchen @ 2017-01-26 16:07 UTC (permalink / raw)
To: herbert, davem, nicolas.ferre
Cc: smueller, Cyrille Pitchen, linux-crypto, linux-arm-kernel,
linux-kernel
In-Reply-To: <cover.1485443478.git.cyrille.pitchen@atmel.com>
This patch is a transitional patch. It updates atmel_sha_done_task() to
make it more generic. Indeed, it adds a new .resume() member in the
atmel_sha_dev structure. This hook is called from atmel_sha_done_task()
to resume processing an asynchronous request.
Signed-off-by: Cyrille Pitchen <cyrille.pitchen@atmel.com>
---
drivers/crypto/atmel-sha.c | 21 ++++++++++++++++-----
1 file changed, 16 insertions(+), 5 deletions(-)
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index 2dbed8bb8d26..643d79a05dda 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -138,6 +138,7 @@ struct atmel_sha_dev {
struct crypto_queue queue;
struct ahash_request *req;
bool is_async;
+ atmel_sha_fn_t resume;
struct atmel_sha_dma dma_lch_in;
@@ -946,6 +947,8 @@ static int atmel_sha_handle_queue(struct atmel_sha_dev *dd,
return (start_async) ? ret : err;
}
+static int atmel_sha_done(struct atmel_sha_dev *dd);
+
static int atmel_sha_start(struct atmel_sha_dev *dd)
{
struct ahash_request *req = dd->req;
@@ -960,6 +963,7 @@ static int atmel_sha_start(struct atmel_sha_dev *dd)
if (err)
goto err1;
+ dd->resume = atmel_sha_done;
if (ctx->op == SHA_OP_UPDATE) {
err = atmel_sha_update_req(dd);
if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP))
@@ -1215,13 +1219,10 @@ static void atmel_sha_queue_task(unsigned long data)
atmel_sha_handle_queue(dd, NULL);
}
-static void atmel_sha_done_task(unsigned long data)
+static int atmel_sha_done(struct atmel_sha_dev *dd)
{
- struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data;
int err = 0;
- dd->is_async = true;
-
if (SHA_FLAGS_CPU & dd->flags) {
if (SHA_FLAGS_OUTPUT_READY & dd->flags) {
dd->flags &= ~SHA_FLAGS_OUTPUT_READY;
@@ -1245,11 +1246,21 @@ static void atmel_sha_done_task(unsigned long data)
goto finish;
}
}
- return;
+ return err;
finish:
/* finish curent request */
atmel_sha_finish_req(dd->req, err);
+
+ return err;
+}
+
+static void atmel_sha_done_task(unsigned long data)
+{
+ struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data;
+
+ dd->is_async = true;
+ (void)dd->resume(dd);
}
static irqreturn_t atmel_sha_irq(int irq, void *dev_id)
--
2.7.4
^ permalink raw reply related
* [PATCH v3 04/12] crypto: atmel-sha: redefine SHA_FLAGS_SHA* flags to match SHA_MR_ALGO_SHA*
From: Cyrille Pitchen @ 2017-01-26 16:07 UTC (permalink / raw)
To: herbert, davem, nicolas.ferre
Cc: smueller, Cyrille Pitchen, linux-crypto, linux-arm-kernel,
linux-kernel
In-Reply-To: <cover.1485443478.git.cyrille.pitchen@atmel.com>
This patch modifies the SHA_FLAGS_SHA* flags: those algo flags are now
organized as values of a single bitfield instead of individual bits.
This allows to reduce the number of bits needed to encode all possible
values. Also the new values match the SHA_MR_ALGO_SHA* values hence
the algorithm bitfield of the SHA_MR register could simply be set with:
mr = (mr & ~SHA_FLAGS_ALGO_MASK) | (ctx->flags & SHA_FLAGS_ALGO_MASK)
Signed-off-by: Cyrille Pitchen <cyrille.pitchen@atmel.com>
---
drivers/crypto/atmel-sha-regs.h | 1 +
drivers/crypto/atmel-sha.c | 45 +++++++++++++++++++++++++++++------------
2 files changed, 33 insertions(+), 13 deletions(-)
diff --git a/drivers/crypto/atmel-sha-regs.h b/drivers/crypto/atmel-sha-regs.h
index e08897109cab..deb0b0b15096 100644
--- a/drivers/crypto/atmel-sha-regs.h
+++ b/drivers/crypto/atmel-sha-regs.h
@@ -19,6 +19,7 @@
#define SHA_MR_PROCDLY (1 << 4)
#define SHA_MR_UIHV (1 << 5)
#define SHA_MR_UIEHV (1 << 6)
+#define SHA_MR_ALGO_MASK GENMASK(10, 8)
#define SHA_MR_ALGO_SHA1 (0 << 8)
#define SHA_MR_ALGO_SHA256 (1 << 8)
#define SHA_MR_ALGO_SHA384 (2 << 8)
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index 643d79a05dda..b29a4e5bc404 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -51,14 +51,16 @@
#define SHA_FLAGS_CPU BIT(5)
#define SHA_FLAGS_DMA_READY BIT(6)
+/* bits[10:8] are reserved. */
+#define SHA_FLAGS_ALGO_MASK SHA_MR_ALGO_MASK
+#define SHA_FLAGS_SHA1 SHA_MR_ALGO_SHA1
+#define SHA_FLAGS_SHA256 SHA_MR_ALGO_SHA256
+#define SHA_FLAGS_SHA384 SHA_MR_ALGO_SHA384
+#define SHA_FLAGS_SHA512 SHA_MR_ALGO_SHA512
+#define SHA_FLAGS_SHA224 SHA_MR_ALGO_SHA224
+
#define SHA_FLAGS_FINUP BIT(16)
#define SHA_FLAGS_SG BIT(17)
-#define SHA_FLAGS_ALGO_MASK GENMASK(22, 18)
-#define SHA_FLAGS_SHA1 BIT(18)
-#define SHA_FLAGS_SHA224 BIT(19)
-#define SHA_FLAGS_SHA256 BIT(20)
-#define SHA_FLAGS_SHA384 BIT(21)
-#define SHA_FLAGS_SHA512 BIT(22)
#define SHA_FLAGS_ERROR BIT(23)
#define SHA_FLAGS_PAD BIT(24)
#define SHA_FLAGS_RESTORE BIT(25)
@@ -264,7 +266,9 @@ static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length)
bits[1] = cpu_to_be64(size[0] << 3);
bits[0] = cpu_to_be64(size[1] << 3 | size[0] >> 61);
- if (ctx->flags & (SHA_FLAGS_SHA384 | SHA_FLAGS_SHA512)) {
+ switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {
+ case SHA_FLAGS_SHA384:
+ case SHA_FLAGS_SHA512:
index = ctx->bufcnt & 0x7f;
padlen = (index < 112) ? (112 - index) : ((128+112) - index);
*(ctx->buffer + ctx->bufcnt) = 0x80;
@@ -272,7 +276,9 @@ static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length)
memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16);
ctx->bufcnt += padlen + 16;
ctx->flags |= SHA_FLAGS_PAD;
- } else {
+ break;
+
+ default:
index = ctx->bufcnt & 0x3f;
padlen = (index < 56) ? (56 - index) : ((64+56) - index);
*(ctx->buffer + ctx->bufcnt) = 0x80;
@@ -280,6 +286,7 @@ static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length)
memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8);
ctx->bufcnt += padlen + 8;
ctx->flags |= SHA_FLAGS_PAD;
+ break;
}
}
@@ -828,16 +835,28 @@ static void atmel_sha_copy_ready_hash(struct ahash_request *req)
if (!req->result)
return;
- if (ctx->flags & SHA_FLAGS_SHA1)
+ switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {
+ default:
+ case SHA_FLAGS_SHA1:
memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE);
- else if (ctx->flags & SHA_FLAGS_SHA224)
+ break;
+
+ case SHA_FLAGS_SHA224:
memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE);
- else if (ctx->flags & SHA_FLAGS_SHA256)
+ break;
+
+ case SHA_FLAGS_SHA256:
memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE);
- else if (ctx->flags & SHA_FLAGS_SHA384)
+ break;
+
+ case SHA_FLAGS_SHA384:
memcpy(req->result, ctx->digest, SHA384_DIGEST_SIZE);
- else
+ break;
+
+ case SHA_FLAGS_SHA512:
memcpy(req->result, ctx->digest, SHA512_DIGEST_SIZE);
+ break;
+ }
}
static int atmel_sha_finish(struct ahash_request *req)
--
2.7.4
^ permalink raw reply related
* [PATCH v3 06/12] crypto: atmel-sha: add SHA_MR_MODE_IDATAR0
From: Cyrille Pitchen @ 2017-01-26 16:07 UTC (permalink / raw)
To: herbert, davem, nicolas.ferre
Cc: smueller, linux-crypto, linux-kernel, linux-arm-kernel,
Cyrille Pitchen
In-Reply-To: <cover.1485443478.git.cyrille.pitchen@atmel.com>
This patch defines an alias macro to SHA_MR_MODE_PDC, which is not suited
for DMA usage.
Signed-off-by: Cyrille Pitchen <cyrille.pitchen@atmel.com>
---
drivers/crypto/atmel-sha-regs.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/crypto/atmel-sha-regs.h b/drivers/crypto/atmel-sha-regs.h
index deb0b0b15096..8d62d31eda08 100644
--- a/drivers/crypto/atmel-sha-regs.h
+++ b/drivers/crypto/atmel-sha-regs.h
@@ -16,6 +16,7 @@
#define SHA_MR_MODE_MANUAL 0x0
#define SHA_MR_MODE_AUTO 0x1
#define SHA_MR_MODE_PDC 0x2
+#define SHA_MR_MODE_IDATAR0 0x2
#define SHA_MR_PROCDLY (1 << 4)
#define SHA_MR_UIHV (1 << 5)
#define SHA_MR_UIEHV (1 << 6)
--
2.7.4
^ permalink raw reply related
* [PATCH v3 07/12] crypto: atmel-sha: add atmel_sha_cpu_start()
From: Cyrille Pitchen @ 2017-01-26 16:07 UTC (permalink / raw)
To: herbert, davem, nicolas.ferre
Cc: smueller, Cyrille Pitchen, linux-crypto, linux-arm-kernel,
linux-kernel
In-Reply-To: <cover.1485443478.git.cyrille.pitchen@atmel.com>
This patch adds a simple function to perform data transfer with PIO, hence
handled by the CPU.
Signed-off-by: Cyrille Pitchen <cyrille.pitchen@atmel.com>
---
drivers/crypto/atmel-sha.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 90 insertions(+)
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index be0d72cf4352..58d9ca8ac0f2 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -64,6 +64,8 @@
#define SHA_FLAGS_ERROR BIT(23)
#define SHA_FLAGS_PAD BIT(24)
#define SHA_FLAGS_RESTORE BIT(25)
+#define SHA_FLAGS_IDATAR0 BIT(26)
+#define SHA_FLAGS_WAIT_DATARDY BIT(27)
#define SHA_OP_UPDATE 1
#define SHA_OP_FINAL 2
@@ -141,6 +143,7 @@ struct atmel_sha_dev {
struct ahash_request *req;
bool is_async;
atmel_sha_fn_t resume;
+ atmel_sha_fn_t cpu_transfer_complete;
struct atmel_sha_dma dma_lch_in;
@@ -1317,6 +1320,93 @@ static irqreturn_t atmel_sha_irq(int irq, void *dev_id)
return IRQ_NONE;
}
+
+/* CPU transfer functions */
+
+static int atmel_sha_cpu_transfer(struct atmel_sha_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
+ const u32 *words = (const u32 *)ctx->buffer;
+ size_t i, num_words;
+ u32 isr, din, din_inc;
+
+ din_inc = (ctx->flags & SHA_FLAGS_IDATAR0) ? 0 : 1;
+ for (;;) {
+ /* Write data into the Input Data Registers. */
+ num_words = DIV_ROUND_UP(ctx->bufcnt, sizeof(u32));
+ for (i = 0, din = 0; i < num_words; ++i, din += din_inc)
+ atmel_sha_write(dd, SHA_REG_DIN(din), words[i]);
+
+ ctx->offset += ctx->bufcnt;
+ ctx->total -= ctx->bufcnt;
+
+ if (!ctx->total)
+ break;
+
+ /*
+ * Prepare next block:
+ * Fill ctx->buffer now with the next data to be written into
+ * IDATARx: it gives time for the SHA hardware to process
+ * the current data so the SHA_INT_DATARDY flag might be set
+ * in SHA_ISR when polling this register at the beginning of
+ * the next loop.
+ */
+ ctx->bufcnt = min_t(size_t, ctx->block_size, ctx->total);
+ scatterwalk_map_and_copy(ctx->buffer, ctx->sg,
+ ctx->offset, ctx->bufcnt, 0);
+
+ /* Wait for hardware to be ready again. */
+ isr = atmel_sha_read(dd, SHA_ISR);
+ if (!(isr & SHA_INT_DATARDY)) {
+ /* Not ready yet. */
+ dd->resume = atmel_sha_cpu_transfer;
+ atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);
+ return -EINPROGRESS;
+ }
+ }
+
+ if (unlikely(!(ctx->flags & SHA_FLAGS_WAIT_DATARDY)))
+ return dd->cpu_transfer_complete(dd);
+
+ return atmel_sha_wait_for_data_ready(dd, dd->cpu_transfer_complete);
+}
+
+static int atmel_sha_cpu_start(struct atmel_sha_dev *dd,
+ struct scatterlist *sg,
+ unsigned int len,
+ bool idatar0_only,
+ bool wait_data_ready,
+ atmel_sha_fn_t resume)
+{
+ struct ahash_request *req = dd->req;
+ struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
+
+ if (!len)
+ return resume(dd);
+
+ ctx->flags &= ~(SHA_FLAGS_IDATAR0 | SHA_FLAGS_WAIT_DATARDY);
+
+ if (idatar0_only)
+ ctx->flags |= SHA_FLAGS_IDATAR0;
+
+ if (wait_data_ready)
+ ctx->flags |= SHA_FLAGS_WAIT_DATARDY;
+
+ ctx->sg = sg;
+ ctx->total = len;
+ ctx->offset = 0;
+
+ /* Prepare the first block to be written. */
+ ctx->bufcnt = min_t(size_t, ctx->block_size, ctx->total);
+ scatterwalk_map_and_copy(ctx->buffer, ctx->sg,
+ ctx->offset, ctx->bufcnt, 0);
+
+ dd->cpu_transfer_complete = resume;
+ return atmel_sha_cpu_transfer(dd);
+}
+
+
static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd)
{
int i;
--
2.7.4
^ permalink raw reply related
* [PATCH v3 08/12] crypto: atmel-sha: add simple DMA transfers
From: Cyrille Pitchen @ 2017-01-26 16:07 UTC (permalink / raw)
To: herbert, davem, nicolas.ferre
Cc: smueller, Cyrille Pitchen, linux-crypto, linux-arm-kernel,
linux-kernel
In-Reply-To: <cover.1485443478.git.cyrille.pitchen@atmel.com>
This patch adds a simple function to perform data transfer with the DMA
controller.
Signed-off-by: Cyrille Pitchen <cyrille.pitchen@atmel.com>
---
drivers/crypto/atmel-sha.c | 116 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 116 insertions(+)
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index 58d9ca8ac0f2..a4fc60b67099 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -123,6 +123,9 @@ struct atmel_sha_ctx {
struct atmel_sha_dma {
struct dma_chan *chan;
struct dma_slave_config dma_conf;
+ struct scatterlist *sg;
+ int nents;
+ unsigned int last_sg_length;
};
struct atmel_sha_dev {
@@ -1321,6 +1324,119 @@ static irqreturn_t atmel_sha_irq(int irq, void *dev_id)
}
+/* DMA transfer functions */
+
+static bool atmel_sha_dma_check_aligned(struct atmel_sha_dev *dd,
+ struct scatterlist *sg,
+ size_t len)
+{
+ struct atmel_sha_dma *dma = &dd->dma_lch_in;
+ struct ahash_request *req = dd->req;
+ struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
+ size_t bs = ctx->block_size;
+ int nents;
+
+ for (nents = 0; sg; sg = sg_next(sg), ++nents) {
+ if (!IS_ALIGNED(sg->offset, sizeof(u32)))
+ return false;
+
+ /*
+ * This is the last sg, the only one that is allowed to
+ * have an unaligned length.
+ */
+ if (len <= sg->length) {
+ dma->nents = nents + 1;
+ dma->last_sg_length = sg->length;
+ sg->length = ALIGN(len, sizeof(u32));
+ return true;
+ }
+
+ /* All other sg lengths MUST be aligned to the block size. */
+ if (!IS_ALIGNED(sg->length, bs))
+ return false;
+
+ len -= sg->length;
+ }
+
+ return false;
+}
+
+static void atmel_sha_dma_callback2(void *data)
+{
+ struct atmel_sha_dev *dd = data;
+ struct atmel_sha_dma *dma = &dd->dma_lch_in;
+ struct scatterlist *sg;
+ int nents;
+
+ dmaengine_terminate_all(dma->chan);
+ dma_unmap_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE);
+
+ sg = dma->sg;
+ for (nents = 0; nents < dma->nents - 1; ++nents)
+ sg = sg_next(sg);
+ sg->length = dma->last_sg_length;
+
+ dd->is_async = true;
+ (void)atmel_sha_wait_for_data_ready(dd, dd->resume);
+}
+
+static int atmel_sha_dma_start(struct atmel_sha_dev *dd,
+ struct scatterlist *src,
+ size_t len,
+ atmel_sha_fn_t resume)
+{
+ struct atmel_sha_dma *dma = &dd->dma_lch_in;
+ struct dma_slave_config *config = &dma->dma_conf;
+ struct dma_chan *chan = dma->chan;
+ struct dma_async_tx_descriptor *desc;
+ dma_cookie_t cookie;
+ unsigned int sg_len;
+ int err;
+
+ dd->resume = resume;
+
+ /*
+ * dma->nents has already been initialized by
+ * atmel_sha_dma_check_aligned().
+ */
+ dma->sg = src;
+ sg_len = dma_map_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE);
+ if (!sg_len) {
+ err = -ENOMEM;
+ goto exit;
+ }
+
+ config->src_maxburst = 16;
+ config->dst_maxburst = 16;
+ err = dmaengine_slave_config(chan, config);
+ if (err)
+ goto unmap_sg;
+
+ desc = dmaengine_prep_slave_sg(chan, dma->sg, sg_len, DMA_MEM_TO_DEV,
+ DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+ if (!desc) {
+ err = -ENOMEM;
+ goto unmap_sg;
+ }
+
+ desc->callback = atmel_sha_dma_callback2;
+ desc->callback_param = dd;
+ cookie = dmaengine_submit(desc);
+ err = dma_submit_error(cookie);
+ if (err)
+ goto unmap_sg;
+
+ dma_async_issue_pending(chan);
+
+ return -EINPROGRESS;
+
+unmap_sg:
+ dma_unmap_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE);
+exit:
+ return atmel_sha_complete(dd, err);
+}
+
+
/* CPU transfer functions */
static int atmel_sha_cpu_transfer(struct atmel_sha_dev *dd)
--
2.7.4
^ permalink raw reply related
* [PATCH v3 09/12] crypto: atmel-sha: add support to hmac(shaX)
From: Cyrille Pitchen @ 2017-01-26 16:07 UTC (permalink / raw)
To: herbert, davem, nicolas.ferre
Cc: smueller, Cyrille Pitchen, linux-crypto, linux-arm-kernel,
linux-kernel
In-Reply-To: <cover.1485443478.git.cyrille.pitchen@atmel.com>
This patch adds support to the hmac(shaX) algorithms.
Signed-off-by: Cyrille Pitchen <cyrille.pitchen@atmel.com>
---
drivers/crypto/atmel-sha-regs.h | 4 +
drivers/crypto/atmel-sha.c | 598 +++++++++++++++++++++++++++++++++++++++-
2 files changed, 601 insertions(+), 1 deletion(-)
diff --git a/drivers/crypto/atmel-sha-regs.h b/drivers/crypto/atmel-sha-regs.h
index 8d62d31eda08..1b9f3d33079e 100644
--- a/drivers/crypto/atmel-sha-regs.h
+++ b/drivers/crypto/atmel-sha-regs.h
@@ -26,6 +26,7 @@
#define SHA_MR_ALGO_SHA384 (2 << 8)
#define SHA_MR_ALGO_SHA512 (3 << 8)
#define SHA_MR_ALGO_SHA224 (4 << 8)
+#define SHA_MR_HMAC (1 << 11)
#define SHA_MR_DUALBUFF (1 << 16)
#define SHA_IER 0x10
@@ -42,6 +43,9 @@
#define SHA_ISR_URAT_MR (0x2 << 12)
#define SHA_ISR_URAT_WO (0x5 << 12)
+#define SHA_MSR 0x20
+#define SHA_BCR 0x30
+
#define SHA_HW_VERSION 0xFC
#define SHA_TPR 0x108
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index a4fc60b67099..78c3c02e4483 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -51,13 +51,20 @@
#define SHA_FLAGS_CPU BIT(5)
#define SHA_FLAGS_DMA_READY BIT(6)
-/* bits[10:8] are reserved. */
+/* bits[11:8] are reserved. */
#define SHA_FLAGS_ALGO_MASK SHA_MR_ALGO_MASK
#define SHA_FLAGS_SHA1 SHA_MR_ALGO_SHA1
#define SHA_FLAGS_SHA256 SHA_MR_ALGO_SHA256
#define SHA_FLAGS_SHA384 SHA_MR_ALGO_SHA384
#define SHA_FLAGS_SHA512 SHA_MR_ALGO_SHA512
#define SHA_FLAGS_SHA224 SHA_MR_ALGO_SHA224
+#define SHA_FLAGS_HMAC SHA_MR_HMAC
+#define SHA_FLAGS_HMAC_SHA1 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA1)
+#define SHA_FLAGS_HMAC_SHA256 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA256)
+#define SHA_FLAGS_HMAC_SHA384 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA384)
+#define SHA_FLAGS_HMAC_SHA512 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA512)
+#define SHA_FLAGS_HMAC_SHA224 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA224)
+#define SHA_FLAGS_MODE_MASK (SHA_FLAGS_HMAC | SHA_FLAGS_ALGO_MASK)
#define SHA_FLAGS_FINUP BIT(16)
#define SHA_FLAGS_SG BIT(17)
@@ -67,8 +74,10 @@
#define SHA_FLAGS_IDATAR0 BIT(26)
#define SHA_FLAGS_WAIT_DATARDY BIT(27)
+#define SHA_OP_INIT 0
#define SHA_OP_UPDATE 1
#define SHA_OP_FINAL 2
+#define SHA_OP_DIGEST 3
#define SHA_BUFFER_LEN (PAGE_SIZE / 16)
@@ -80,6 +89,7 @@ struct atmel_sha_caps {
bool has_sha224;
bool has_sha_384_512;
bool has_uihv;
+ bool has_hmac;
};
struct atmel_sha_dev;
@@ -105,6 +115,7 @@ struct atmel_sha_reqctx {
unsigned int total; /* total request */
size_t block_size;
+ size_t hash_size;
u8 buffer[SHA_BUFFER_LEN + SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
};
@@ -152,6 +163,8 @@ struct atmel_sha_dev {
struct atmel_sha_caps caps;
+ struct scatterlist tmp;
+
u32 hw_version;
};
@@ -1522,11 +1535,579 @@ static int atmel_sha_cpu_start(struct atmel_sha_dev *dd,
return atmel_sha_cpu_transfer(dd);
}
+static int atmel_sha_cpu_hash(struct atmel_sha_dev *dd,
+ const void *data, unsigned int datalen,
+ bool auto_padding,
+ atmel_sha_fn_t resume)
+{
+ struct ahash_request *req = dd->req;
+ struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
+ u32 msglen = (auto_padding) ? datalen : 0;
+ u32 mr = SHA_MR_MODE_AUTO;
+
+ if (!(IS_ALIGNED(datalen, ctx->block_size) || auto_padding))
+ return atmel_sha_complete(dd, -EINVAL);
+
+ mr |= (ctx->flags & SHA_FLAGS_ALGO_MASK);
+ atmel_sha_write(dd, SHA_MR, mr);
+ atmel_sha_write(dd, SHA_MSR, msglen);
+ atmel_sha_write(dd, SHA_BCR, msglen);
+ atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);
+
+ sg_init_one(&dd->tmp, data, datalen);
+ return atmel_sha_cpu_start(dd, &dd->tmp, datalen, false, true, resume);
+}
+
+
+/* hmac functions */
+
+struct atmel_sha_hmac_key {
+ bool valid;
+ unsigned int keylen;
+ u8 buffer[SHA512_BLOCK_SIZE];
+ u8 *keydup;
+};
+
+static inline void atmel_sha_hmac_key_init(struct atmel_sha_hmac_key *hkey)
+{
+ memset(hkey, 0, sizeof(*hkey));
+}
+
+static inline void atmel_sha_hmac_key_release(struct atmel_sha_hmac_key *hkey)
+{
+ kfree(hkey->keydup);
+ memset(hkey, 0, sizeof(*hkey));
+}
+
+static inline int atmel_sha_hmac_key_set(struct atmel_sha_hmac_key *hkey,
+ const u8 *key,
+ unsigned int keylen)
+{
+ atmel_sha_hmac_key_release(hkey);
+
+ if (keylen > sizeof(hkey->buffer)) {
+ hkey->keydup = kmemdup(key, keylen, GFP_KERNEL);
+ if (!hkey->keydup)
+ return -ENOMEM;
+
+ } else {
+ memcpy(hkey->buffer, key, keylen);
+ }
+
+ hkey->valid = true;
+ hkey->keylen = keylen;
+ return 0;
+}
+
+static inline bool atmel_sha_hmac_key_get(const struct atmel_sha_hmac_key *hkey,
+ const u8 **key,
+ unsigned int *keylen)
+{
+ if (!hkey->valid)
+ return false;
+
+ *keylen = hkey->keylen;
+ *key = (hkey->keydup) ? hkey->keydup : hkey->buffer;
+ return true;
+}
+
+
+struct atmel_sha_hmac_ctx {
+ struct atmel_sha_ctx base;
+
+ struct atmel_sha_hmac_key hkey;
+ u32 ipad[SHA512_BLOCK_SIZE / sizeof(u32)];
+ u32 opad[SHA512_BLOCK_SIZE / sizeof(u32)];
+ atmel_sha_fn_t resume;
+};
+
+static int atmel_sha_hmac_setup(struct atmel_sha_dev *dd,
+ atmel_sha_fn_t resume);
+static int atmel_sha_hmac_prehash_key(struct atmel_sha_dev *dd,
+ const u8 *key, unsigned int keylen);
+static int atmel_sha_hmac_prehash_key_done(struct atmel_sha_dev *dd);
+static int atmel_sha_hmac_compute_ipad_hash(struct atmel_sha_dev *dd);
+static int atmel_sha_hmac_compute_opad_hash(struct atmel_sha_dev *dd);
+static int atmel_sha_hmac_setup_done(struct atmel_sha_dev *dd);
+
+static int atmel_sha_hmac_init_done(struct atmel_sha_dev *dd);
+static int atmel_sha_hmac_final(struct atmel_sha_dev *dd);
+static int atmel_sha_hmac_final_done(struct atmel_sha_dev *dd);
+static int atmel_sha_hmac_digest2(struct atmel_sha_dev *dd);
+
+static int atmel_sha_hmac_setup(struct atmel_sha_dev *dd,
+ atmel_sha_fn_t resume)
+{
+ struct ahash_request *req = dd->req;
+ struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
+ unsigned int keylen;
+ const u8 *key;
+ size_t bs;
+
+ hmac->resume = resume;
+ switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {
+ case SHA_FLAGS_SHA1:
+ ctx->block_size = SHA1_BLOCK_SIZE;
+ ctx->hash_size = SHA1_DIGEST_SIZE;
+ break;
+
+ case SHA_FLAGS_SHA224:
+ ctx->block_size = SHA224_BLOCK_SIZE;
+ ctx->hash_size = SHA256_DIGEST_SIZE;
+ break;
+
+ case SHA_FLAGS_SHA256:
+ ctx->block_size = SHA256_BLOCK_SIZE;
+ ctx->hash_size = SHA256_DIGEST_SIZE;
+ break;
+
+ case SHA_FLAGS_SHA384:
+ ctx->block_size = SHA384_BLOCK_SIZE;
+ ctx->hash_size = SHA512_DIGEST_SIZE;
+ break;
+
+ case SHA_FLAGS_SHA512:
+ ctx->block_size = SHA512_BLOCK_SIZE;
+ ctx->hash_size = SHA512_DIGEST_SIZE;
+ break;
+
+ default:
+ return atmel_sha_complete(dd, -EINVAL);
+ }
+ bs = ctx->block_size;
+
+ if (likely(!atmel_sha_hmac_key_get(&hmac->hkey, &key, &keylen)))
+ return resume(dd);
+
+ /* Compute K' from K. */
+ if (unlikely(keylen > bs))
+ return atmel_sha_hmac_prehash_key(dd, key, keylen);
+
+ /* Prepare ipad. */
+ memcpy((u8 *)hmac->ipad, key, keylen);
+ memset((u8 *)hmac->ipad + keylen, 0, bs - keylen);
+ return atmel_sha_hmac_compute_ipad_hash(dd);
+}
+
+static int atmel_sha_hmac_prehash_key(struct atmel_sha_dev *dd,
+ const u8 *key, unsigned int keylen)
+{
+ return atmel_sha_cpu_hash(dd, key, keylen, true,
+ atmel_sha_hmac_prehash_key_done);
+}
+
+static int atmel_sha_hmac_prehash_key_done(struct atmel_sha_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
+ struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
+ size_t ds = crypto_ahash_digestsize(tfm);
+ size_t bs = ctx->block_size;
+ size_t i, num_words = ds / sizeof(u32);
+
+ /* Prepare ipad. */
+ for (i = 0; i < num_words; ++i)
+ hmac->ipad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));
+ memset((u8 *)hmac->ipad + ds, 0, bs - ds);
+ return atmel_sha_hmac_compute_ipad_hash(dd);
+}
+
+static int atmel_sha_hmac_compute_ipad_hash(struct atmel_sha_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
+ struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
+ size_t bs = ctx->block_size;
+ size_t i, num_words = bs / sizeof(u32);
+
+ memcpy(hmac->opad, hmac->ipad, bs);
+ for (i = 0; i < num_words; ++i) {
+ hmac->ipad[i] ^= 0x36363636;
+ hmac->opad[i] ^= 0x5c5c5c5c;
+ }
+
+ return atmel_sha_cpu_hash(dd, hmac->ipad, bs, false,
+ atmel_sha_hmac_compute_opad_hash);
+}
+
+static int atmel_sha_hmac_compute_opad_hash(struct atmel_sha_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
+ struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
+ size_t bs = ctx->block_size;
+ size_t hs = ctx->hash_size;
+ size_t i, num_words = hs / sizeof(u32);
+
+ for (i = 0; i < num_words; ++i)
+ hmac->ipad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));
+ return atmel_sha_cpu_hash(dd, hmac->opad, bs, false,
+ atmel_sha_hmac_setup_done);
+}
+
+static int atmel_sha_hmac_setup_done(struct atmel_sha_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
+ struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
+ size_t hs = ctx->hash_size;
+ size_t i, num_words = hs / sizeof(u32);
+
+ for (i = 0; i < num_words; ++i)
+ hmac->opad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));
+ atmel_sha_hmac_key_release(&hmac->hkey);
+ return hmac->resume(dd);
+}
+
+static int atmel_sha_hmac_start(struct atmel_sha_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
+ int err;
+
+ err = atmel_sha_hw_init(dd);
+ if (err)
+ return atmel_sha_complete(dd, err);
+
+ switch (ctx->op) {
+ case SHA_OP_INIT:
+ err = atmel_sha_hmac_setup(dd, atmel_sha_hmac_init_done);
+ break;
+
+ case SHA_OP_UPDATE:
+ dd->resume = atmel_sha_done;
+ err = atmel_sha_update_req(dd);
+ break;
+
+ case SHA_OP_FINAL:
+ dd->resume = atmel_sha_hmac_final;
+ err = atmel_sha_final_req(dd);
+ break;
+
+ case SHA_OP_DIGEST:
+ err = atmel_sha_hmac_setup(dd, atmel_sha_hmac_digest2);
+ break;
+
+ default:
+ return atmel_sha_complete(dd, -EINVAL);
+ }
+
+ return err;
+}
+
+static int atmel_sha_hmac_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
+
+ if (atmel_sha_hmac_key_set(&hmac->hkey, key, keylen)) {
+ crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int atmel_sha_hmac_init(struct ahash_request *req)
+{
+ int err;
+
+ err = atmel_sha_init(req);
+ if (err)
+ return err;
+
+ return atmel_sha_enqueue(req, SHA_OP_INIT);
+}
+
+static int atmel_sha_hmac_init_done(struct atmel_sha_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
+ size_t bs = ctx->block_size;
+ size_t hs = ctx->hash_size;
+
+ ctx->bufcnt = 0;
+ ctx->digcnt[0] = bs;
+ ctx->digcnt[1] = 0;
+ ctx->flags |= SHA_FLAGS_RESTORE;
+ memcpy(ctx->digest, hmac->ipad, hs);
+ return atmel_sha_complete(dd, 0);
+}
+
+static int atmel_sha_hmac_final(struct atmel_sha_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
+ u32 *digest = (u32 *)ctx->digest;
+ size_t ds = crypto_ahash_digestsize(tfm);
+ size_t bs = ctx->block_size;
+ size_t hs = ctx->hash_size;
+ size_t i, num_words;
+ u32 mr;
+
+ /* Save d = SHA((K' + ipad) | msg). */
+ num_words = ds / sizeof(u32);
+ for (i = 0; i < num_words; ++i)
+ digest[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));
+
+ /* Restore context to finish computing SHA((K' + opad) | d). */
+ atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV);
+ num_words = hs / sizeof(u32);
+ for (i = 0; i < num_words; ++i)
+ atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]);
+
+ mr = SHA_MR_MODE_AUTO | SHA_MR_UIHV;
+ mr |= (ctx->flags & SHA_FLAGS_ALGO_MASK);
+ atmel_sha_write(dd, SHA_MR, mr);
+ atmel_sha_write(dd, SHA_MSR, bs + ds);
+ atmel_sha_write(dd, SHA_BCR, ds);
+ atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);
+
+ sg_init_one(&dd->tmp, digest, ds);
+ return atmel_sha_cpu_start(dd, &dd->tmp, ds, false, true,
+ atmel_sha_hmac_final_done);
+}
+
+static int atmel_sha_hmac_final_done(struct atmel_sha_dev *dd)
+{
+ /*
+ * req->result might not be sizeof(u32) aligned, so copy the
+ * digest into ctx->digest[] before memcpy() the data into
+ * req->result.
+ */
+ atmel_sha_copy_hash(dd->req);
+ atmel_sha_copy_ready_hash(dd->req);
+ return atmel_sha_complete(dd, 0);
+}
+
+static int atmel_sha_hmac_digest(struct ahash_request *req)
+{
+ int err;
+
+ err = atmel_sha_init(req);
+ if (err)
+ return err;
+
+ return atmel_sha_enqueue(req, SHA_OP_DIGEST);
+}
+
+static int atmel_sha_hmac_digest2(struct atmel_sha_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
+ size_t hs = ctx->hash_size;
+ size_t i, num_words = hs / sizeof(u32);
+ bool use_dma = false;
+ u32 mr;
+
+ /* Special case for empty message. */
+ if (!req->nbytes)
+ return atmel_sha_complete(dd, -EINVAL); // TODO:
+
+ /* Check DMA threshold and alignment. */
+ if (req->nbytes > ATMEL_SHA_DMA_THRESHOLD &&
+ atmel_sha_dma_check_aligned(dd, req->src, req->nbytes))
+ use_dma = true;
+
+ /* Write both initial hash values to compute a HMAC. */
+ atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV);
+ for (i = 0; i < num_words; ++i)
+ atmel_sha_write(dd, SHA_REG_DIN(i), hmac->ipad[i]);
+
+ atmel_sha_write(dd, SHA_CR, SHA_CR_WUIEHV);
+ for (i = 0; i < num_words; ++i)
+ atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]);
+
+ /* Write the Mode, Message Size, Bytes Count then Control Registers. */
+ mr = (SHA_MR_HMAC | SHA_MR_DUALBUFF);
+ mr |= ctx->flags & SHA_FLAGS_ALGO_MASK;
+ if (use_dma)
+ mr |= SHA_MR_MODE_IDATAR0;
+ else
+ mr |= SHA_MR_MODE_AUTO;
+ atmel_sha_write(dd, SHA_MR, mr);
+
+ atmel_sha_write(dd, SHA_MSR, req->nbytes);
+ atmel_sha_write(dd, SHA_BCR, req->nbytes);
+
+ atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);
+
+ /* Process data. */
+ if (use_dma)
+ return atmel_sha_dma_start(dd, req->src, req->nbytes,
+ atmel_sha_hmac_final_done);
+
+ return atmel_sha_cpu_start(dd, req->src, req->nbytes, false, true,
+ atmel_sha_hmac_final_done);
+}
+
+static int atmel_sha_hmac_cra_init(struct crypto_tfm *tfm)
+{
+ struct atmel_sha_hmac_ctx *hmac = crypto_tfm_ctx(tfm);
+
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct atmel_sha_reqctx));
+ hmac->base.start = atmel_sha_hmac_start;
+ atmel_sha_hmac_key_init(&hmac->hkey);
+
+ return 0;
+}
+
+static void atmel_sha_hmac_cra_exit(struct crypto_tfm *tfm)
+{
+ struct atmel_sha_hmac_ctx *hmac = crypto_tfm_ctx(tfm);
+
+ atmel_sha_hmac_key_release(&hmac->hkey);
+}
+
+static struct ahash_alg sha_hmac_algs[] = {
+{
+ .init = atmel_sha_hmac_init,
+ .update = atmel_sha_update,
+ .final = atmel_sha_final,
+ .digest = atmel_sha_hmac_digest,
+ .setkey = atmel_sha_hmac_setkey,
+ .export = atmel_sha_export,
+ .import = atmel_sha_import,
+ .halg = {
+ .digestsize = SHA1_DIGEST_SIZE,
+ .statesize = sizeof(struct atmel_sha_reqctx),
+ .base = {
+ .cra_name = "hmac(sha1)",
+ .cra_driver_name = "atmel-hmac-sha1",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = atmel_sha_hmac_cra_init,
+ .cra_exit = atmel_sha_hmac_cra_exit,
+ }
+ }
+},
+{
+ .init = atmel_sha_hmac_init,
+ .update = atmel_sha_update,
+ .final = atmel_sha_final,
+ .digest = atmel_sha_hmac_digest,
+ .setkey = atmel_sha_hmac_setkey,
+ .export = atmel_sha_export,
+ .import = atmel_sha_import,
+ .halg = {
+ .digestsize = SHA224_DIGEST_SIZE,
+ .statesize = sizeof(struct atmel_sha_reqctx),
+ .base = {
+ .cra_name = "hmac(sha224)",
+ .cra_driver_name = "atmel-hmac-sha224",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SHA224_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = atmel_sha_hmac_cra_init,
+ .cra_exit = atmel_sha_hmac_cra_exit,
+ }
+ }
+},
+{
+ .init = atmel_sha_hmac_init,
+ .update = atmel_sha_update,
+ .final = atmel_sha_final,
+ .digest = atmel_sha_hmac_digest,
+ .setkey = atmel_sha_hmac_setkey,
+ .export = atmel_sha_export,
+ .import = atmel_sha_import,
+ .halg = {
+ .digestsize = SHA256_DIGEST_SIZE,
+ .statesize = sizeof(struct atmel_sha_reqctx),
+ .base = {
+ .cra_name = "hmac(sha256)",
+ .cra_driver_name = "atmel-hmac-sha256",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = atmel_sha_hmac_cra_init,
+ .cra_exit = atmel_sha_hmac_cra_exit,
+ }
+ }
+},
+{
+ .init = atmel_sha_hmac_init,
+ .update = atmel_sha_update,
+ .final = atmel_sha_final,
+ .digest = atmel_sha_hmac_digest,
+ .setkey = atmel_sha_hmac_setkey,
+ .export = atmel_sha_export,
+ .import = atmel_sha_import,
+ .halg = {
+ .digestsize = SHA384_DIGEST_SIZE,
+ .statesize = sizeof(struct atmel_sha_reqctx),
+ .base = {
+ .cra_name = "hmac(sha384)",
+ .cra_driver_name = "atmel-hmac-sha384",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = atmel_sha_hmac_cra_init,
+ .cra_exit = atmel_sha_hmac_cra_exit,
+ }
+ }
+},
+{
+ .init = atmel_sha_hmac_init,
+ .update = atmel_sha_update,
+ .final = atmel_sha_final,
+ .digest = atmel_sha_hmac_digest,
+ .setkey = atmel_sha_hmac_setkey,
+ .export = atmel_sha_export,
+ .import = atmel_sha_import,
+ .halg = {
+ .digestsize = SHA512_DIGEST_SIZE,
+ .statesize = sizeof(struct atmel_sha_reqctx),
+ .base = {
+ .cra_name = "hmac(sha512)",
+ .cra_driver_name = "atmel-hmac-sha512",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = atmel_sha_hmac_cra_init,
+ .cra_exit = atmel_sha_hmac_cra_exit,
+ }
+ }
+},
+};
static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd)
{
int i;
+ if (dd->caps.has_hmac)
+ for (i = 0; i < ARRAY_SIZE(sha_hmac_algs); i++)
+ crypto_unregister_ahash(&sha_hmac_algs[i]);
+
for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++)
crypto_unregister_ahash(&sha_1_256_algs[i]);
@@ -1563,8 +2144,21 @@ static int atmel_sha_register_algs(struct atmel_sha_dev *dd)
}
}
+ if (dd->caps.has_hmac) {
+ for (i = 0; i < ARRAY_SIZE(sha_hmac_algs); i++) {
+ err = crypto_register_ahash(&sha_hmac_algs[i]);
+ if (err)
+ goto err_sha_hmac_algs;
+ }
+ }
+
return 0;
+ /*i = ARRAY_SIZE(sha_hmac_algs);*/
+err_sha_hmac_algs:
+ for (j = 0; j < i; j++)
+ crypto_unregister_ahash(&sha_hmac_algs[j]);
+ i = ARRAY_SIZE(sha_384_512_algs);
err_sha_384_512_algs:
for (j = 0; j < i; j++)
crypto_unregister_ahash(&sha_384_512_algs[j]);
@@ -1634,6 +2228,7 @@ static void atmel_sha_get_cap(struct atmel_sha_dev *dd)
dd->caps.has_sha224 = 0;
dd->caps.has_sha_384_512 = 0;
dd->caps.has_uihv = 0;
+ dd->caps.has_hmac = 0;
/* keep only major version number */
switch (dd->hw_version & 0xff0) {
@@ -1643,6 +2238,7 @@ static void atmel_sha_get_cap(struct atmel_sha_dev *dd)
dd->caps.has_sha224 = 1;
dd->caps.has_sha_384_512 = 1;
dd->caps.has_uihv = 1;
+ dd->caps.has_hmac = 1;
break;
case 0x420:
dd->caps.has_dma = 1;
--
2.7.4
^ permalink raw reply related
* [PATCH v3 10/12] crypto: atmel-aes: fix atmel_aes_handle_queue()
From: Cyrille Pitchen @ 2017-01-26 16:07 UTC (permalink / raw)
To: herbert, davem, nicolas.ferre
Cc: smueller, Cyrille Pitchen, linux-crypto, linux-arm-kernel,
linux-kernel
In-Reply-To: <cover.1485443478.git.cyrille.pitchen@atmel.com>
This patch fixes the value returned by atmel_aes_handle_queue(), which
could have been wrong previously when the crypto request was started
synchronously but became asynchronous during the ctx->start() call.
Signed-off-by: Cyrille Pitchen <cyrille.pitchen@atmel.com>
---
drivers/crypto/atmel-aes.c | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index 0e3d0d655b96..9fd2f63b8bc0 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -879,6 +879,7 @@ static int atmel_aes_handle_queue(struct atmel_aes_dev *dd,
struct crypto_async_request *areq, *backlog;
struct atmel_aes_base_ctx *ctx;
unsigned long flags;
+ bool start_async;
int err, ret = 0;
spin_lock_irqsave(&dd->lock, flags);
@@ -904,10 +905,12 @@ static int atmel_aes_handle_queue(struct atmel_aes_dev *dd,
dd->areq = areq;
dd->ctx = ctx;
- dd->is_async = (areq != new_areq);
+ start_async = (areq != new_areq);
+ dd->is_async = start_async;
+ /* WARNING: ctx->start() MAY change dd->is_async. */
err = ctx->start(dd);
- return (dd->is_async) ? ret : err;
+ return (start_async) ? ret : err;
}
--
2.7.4
^ permalink raw reply related
* [PATCH v3 11/12] crypto: atmel-authenc: add support to authenc(hmac(shaX), Y(aes)) modes
From: Cyrille Pitchen @ 2017-01-26 16:07 UTC (permalink / raw)
To: herbert, davem, nicolas.ferre
Cc: smueller, Cyrille Pitchen, linux-crypto, linux-arm-kernel,
linux-kernel
In-Reply-To: <cover.1485443478.git.cyrille.pitchen@atmel.com>
This patchs allows to combine the AES and SHA hardware accelerators on
some Atmel SoCs. Doing so, AES blocks are only written to/read from the
AES hardware. Those blocks are also transferred from the AES to the SHA
accelerator internally, without additionnal accesses to the system busses.
Hence, the AES and SHA accelerators work in parallel to process all the
data blocks, instead of serializing the process by (de)crypting those
blocks first then authenticating them after like the generic
crypto/authenc.c driver does.
Of course, both the AES and SHA hardware accelerators need to be available
before we can start to process the data blocks. Hence we use their crypto
request queue to synchronize both drivers.
Signed-off-by: Cyrille Pitchen <cyrille.pitchen@atmel.com>
---
drivers/crypto/Kconfig | 12 ++
drivers/crypto/atmel-aes-regs.h | 16 ++
drivers/crypto/atmel-aes.c | 448 +++++++++++++++++++++++++++++++++++++++-
drivers/crypto/atmel-authenc.h | 64 ++++++
drivers/crypto/atmel-sha-regs.h | 14 ++
drivers/crypto/atmel-sha.c | 344 ++++++++++++++++++++++++++++--
6 files changed, 883 insertions(+), 15 deletions(-)
create mode 100644 drivers/crypto/atmel-authenc.h
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index bf7da55cffe6..74824612d3e9 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -415,6 +415,18 @@ config CRYPTO_DEV_BFIN_CRC
Newer Blackfin processors have CRC hardware. Select this if you
want to use the Blackfin CRC module.
+config CRYPTO_DEV_ATMEL_AUTHENC
+ tristate "Support for Atmel IPSEC/SSL hw accelerator"
+ depends on (ARCH_AT91 && HAS_DMA) || COMPILE_TEST
+ select CRYPTO_AUTHENC
+ select CRYPTO_DEV_ATMEL_AES
+ select CRYPTO_DEV_ATMEL_SHA
+ help
+ Some Atmel processors can combine the AES and SHA hw accelerators
+ to enhance support of IPSEC/SSL.
+ Select this if you want to use the Atmel modules for
+ authenc(hmac(shaX),Y(cbc)) algorithms.
+
config CRYPTO_DEV_ATMEL_AES
tristate "Support for Atmel AES hw accelerator"
depends on HAS_DMA
diff --git a/drivers/crypto/atmel-aes-regs.h b/drivers/crypto/atmel-aes-regs.h
index 0ec04407b533..7694679802b3 100644
--- a/drivers/crypto/atmel-aes-regs.h
+++ b/drivers/crypto/atmel-aes-regs.h
@@ -68,6 +68,22 @@
#define AES_CTRR 0x98
#define AES_GCMHR(x) (0x9c + ((x) * 0x04))
+#define AES_EMR 0xb0
+#define AES_EMR_APEN BIT(0) /* Auto Padding Enable */
+#define AES_EMR_APM BIT(1) /* Auto Padding Mode */
+#define AES_EMR_APM_IPSEC 0x0
+#define AES_EMR_APM_SSL BIT(1)
+#define AES_EMR_PLIPEN BIT(4) /* PLIP Enable */
+#define AES_EMR_PLIPD BIT(5) /* PLIP Decipher */
+#define AES_EMR_PADLEN_MASK (0xFu << 8)
+#define AES_EMR_PADLEN_OFFSET 8
+#define AES_EMR_PADLEN(padlen) (((padlen) << AES_EMR_PADLEN_OFFSET) &\
+ AES_EMR_PADLEN_MASK)
+#define AES_EMR_NHEAD_MASK (0xFu << 16)
+#define AES_EMR_NHEAD_OFFSET 16
+#define AES_EMR_NHEAD(nhead) (((nhead) << AES_EMR_NHEAD_OFFSET) &\
+ AES_EMR_NHEAD_MASK)
+
#define AES_TWR(x) (0xc0 + ((x) * 0x04))
#define AES_ALPHAR(x) (0xd0 + ((x) * 0x04))
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index 9fd2f63b8bc0..29e20c37f3a6 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -41,6 +41,7 @@
#include <linux/platform_data/crypto-atmel.h>
#include <dt-bindings/dma/at91.h>
#include "atmel-aes-regs.h"
+#include "atmel-authenc.h"
#define ATMEL_AES_PRIORITY 300
@@ -78,6 +79,7 @@
#define AES_FLAGS_INIT BIT(2)
#define AES_FLAGS_BUSY BIT(3)
#define AES_FLAGS_DUMP_REG BIT(4)
+#define AES_FLAGS_OWN_SHA BIT(5)
#define AES_FLAGS_PERSISTENT (AES_FLAGS_INIT | AES_FLAGS_BUSY)
@@ -92,6 +94,7 @@ struct atmel_aes_caps {
bool has_ctr32;
bool has_gcm;
bool has_xts;
+ bool has_authenc;
u32 max_burst_size;
};
@@ -144,10 +147,31 @@ struct atmel_aes_xts_ctx {
u32 key2[AES_KEYSIZE_256 / sizeof(u32)];
};
+#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
+struct atmel_aes_authenc_ctx {
+ struct atmel_aes_base_ctx base;
+ struct atmel_sha_authenc_ctx *auth;
+};
+#endif
+
struct atmel_aes_reqctx {
unsigned long mode;
};
+#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
+struct atmel_aes_authenc_reqctx {
+ struct atmel_aes_reqctx base;
+
+ struct scatterlist src[2];
+ struct scatterlist dst[2];
+ size_t textlen;
+ u32 digest[SHA512_DIGEST_SIZE / sizeof(u32)];
+
+ /* auth_req MUST be place last. */
+ struct ahash_request auth_req;
+};
+#endif
+
struct atmel_aes_dma {
struct dma_chan *chan;
struct scatterlist *sg;
@@ -291,6 +315,9 @@ static const char *atmel_aes_reg_name(u32 offset, char *tmp, size_t sz)
snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2);
break;
+ case AES_EMR:
+ return "EMR";
+
case AES_TWR(0):
case AES_TWR(1):
case AES_TWR(2):
@@ -463,8 +490,16 @@ static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd)
return (dd->flags & AES_FLAGS_ENCRYPT);
}
+#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
+static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err);
+#endif
+
static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err)
{
+#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
+ atmel_aes_authenc_complete(dd, err);
+#endif
+
clk_disable(dd->iclk);
dd->flags &= ~AES_FLAGS_BUSY;
@@ -1931,6 +1966,384 @@ static struct crypto_alg aes_xts_alg = {
}
};
+#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
+/* authenc aead functions */
+
+static int atmel_aes_authenc_start(struct atmel_aes_dev *dd);
+static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,
+ bool is_async);
+static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,
+ bool is_async);
+static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd);
+static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,
+ bool is_async);
+
+static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err)
+{
+ struct aead_request *req = aead_request_cast(dd->areq);
+ struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
+
+ if (err && (dd->flags & AES_FLAGS_OWN_SHA))
+ atmel_sha_authenc_abort(&rctx->auth_req);
+ dd->flags &= ~AES_FLAGS_OWN_SHA;
+}
+
+static int atmel_aes_authenc_start(struct atmel_aes_dev *dd)
+{
+ struct aead_request *req = aead_request_cast(dd->areq);
+ struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
+ int err;
+
+ atmel_aes_set_mode(dd, &rctx->base);
+
+ err = atmel_aes_hw_init(dd);
+ if (err)
+ return atmel_aes_complete(dd, err);
+
+ return atmel_sha_authenc_schedule(&rctx->auth_req, ctx->auth,
+ atmel_aes_authenc_init, dd);
+}
+
+static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,
+ bool is_async)
+{
+ struct aead_request *req = aead_request_cast(dd->areq);
+ struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
+
+ if (is_async)
+ dd->is_async = true;
+ if (err)
+ return atmel_aes_complete(dd, err);
+
+ /* If here, we've got the ownership of the SHA device. */
+ dd->flags |= AES_FLAGS_OWN_SHA;
+
+ /* Configure the SHA device. */
+ return atmel_sha_authenc_init(&rctx->auth_req,
+ req->src, req->assoclen,
+ rctx->textlen,
+ atmel_aes_authenc_transfer, dd);
+}
+
+static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,
+ bool is_async)
+{
+ struct aead_request *req = aead_request_cast(dd->areq);
+ struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
+ bool enc = atmel_aes_is_encrypt(dd);
+ struct scatterlist *src, *dst;
+ u32 iv[AES_BLOCK_SIZE / sizeof(u32)];
+ u32 emr;
+
+ if (is_async)
+ dd->is_async = true;
+ if (err)
+ return atmel_aes_complete(dd, err);
+
+ /* Prepare src and dst scatter-lists to transfer cipher/plain texts. */
+ src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
+ dst = src;
+
+ if (req->src != req->dst)
+ dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
+
+ /* Configure the AES device. */
+ memcpy(iv, req->iv, sizeof(iv));
+
+ /*
+ * Here we always set the 2nd parameter of atmel_aes_write_ctrl() to
+ * 'true' even if the data transfer is actually performed by the CPU (so
+ * not by the DMA) because we must force the AES_MR_SMOD bitfield to the
+ * value AES_MR_SMOD_IDATAR0. Indeed, both AES_MR_SMOD and SHA_MR_SMOD
+ * must be set to *_MR_SMOD_IDATAR0.
+ */
+ atmel_aes_write_ctrl(dd, true, iv);
+ emr = AES_EMR_PLIPEN;
+ if (!enc)
+ emr |= AES_EMR_PLIPD;
+ atmel_aes_write(dd, AES_EMR, emr);
+
+ /* Transfer data. */
+ return atmel_aes_dma_start(dd, src, dst, rctx->textlen,
+ atmel_aes_authenc_digest);
+}
+
+static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd)
+{
+ struct aead_request *req = aead_request_cast(dd->areq);
+ struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
+
+ /* atmel_sha_authenc_final() releases the SHA device. */
+ dd->flags &= ~AES_FLAGS_OWN_SHA;
+ return atmel_sha_authenc_final(&rctx->auth_req,
+ rctx->digest, sizeof(rctx->digest),
+ atmel_aes_authenc_final, dd);
+}
+
+static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,
+ bool is_async)
+{
+ struct aead_request *req = aead_request_cast(dd->areq);
+ struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ bool enc = atmel_aes_is_encrypt(dd);
+ u32 idigest[SHA512_DIGEST_SIZE / sizeof(u32)], *odigest = rctx->digest;
+ u32 offs, authsize;
+
+ if (is_async)
+ dd->is_async = true;
+ if (err)
+ goto complete;
+
+ offs = req->assoclen + rctx->textlen;
+ authsize = crypto_aead_authsize(tfm);
+ if (enc) {
+ scatterwalk_map_and_copy(odigest, req->dst, offs, authsize, 1);
+ } else {
+ scatterwalk_map_and_copy(idigest, req->src, offs, authsize, 0);
+ if (crypto_memneq(idigest, odigest, authsize))
+ err = -EBADMSG;
+ }
+
+complete:
+ return atmel_aes_complete(dd, err);
+}
+
+static int atmel_aes_authenc_setkey(struct crypto_aead *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
+ struct crypto_authenc_keys keys;
+ u32 flags;
+ int err;
+
+ if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
+ goto badkey;
+
+ if (keys.enckeylen > sizeof(ctx->base.key))
+ goto badkey;
+
+ /* Save auth key. */
+ flags = crypto_aead_get_flags(tfm);
+ err = atmel_sha_authenc_setkey(ctx->auth,
+ keys.authkey, keys.authkeylen,
+ &flags);
+ crypto_aead_set_flags(tfm, flags & CRYPTO_TFM_RES_MASK);
+ if (err) {
+ memzero_explicit(&keys, sizeof(keys));
+ return err;
+ }
+
+ /* Save enc key. */
+ ctx->base.keylen = keys.enckeylen;
+ memcpy(ctx->base.key, keys.enckey, keys.enckeylen);
+
+ memzero_explicit(&keys, sizeof(keys));
+ return 0;
+
+badkey:
+ crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ memzero_explicit(&key, sizeof(keys));
+ return -EINVAL;
+}
+
+static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm,
+ unsigned long auth_mode)
+{
+ struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
+ unsigned int auth_reqsize = atmel_sha_authenc_get_reqsize();
+
+ ctx->auth = atmel_sha_authenc_spawn(auth_mode);
+ if (IS_ERR(ctx->auth))
+ return PTR_ERR(ctx->auth);
+
+ crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) +
+ auth_reqsize));
+ ctx->base.start = atmel_aes_authenc_start;
+
+ return 0;
+}
+
+static int atmel_aes_authenc_hmac_sha1_init_tfm(struct crypto_aead *tfm)
+{
+ return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA1);
+}
+
+static int atmel_aes_authenc_hmac_sha224_init_tfm(struct crypto_aead *tfm)
+{
+ return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA224);
+}
+
+static int atmel_aes_authenc_hmac_sha256_init_tfm(struct crypto_aead *tfm)
+{
+ return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA256);
+}
+
+static int atmel_aes_authenc_hmac_sha384_init_tfm(struct crypto_aead *tfm)
+{
+ return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA384);
+}
+
+static int atmel_aes_authenc_hmac_sha512_init_tfm(struct crypto_aead *tfm)
+{
+ return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA512);
+}
+
+static void atmel_aes_authenc_exit_tfm(struct crypto_aead *tfm)
+{
+ struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
+
+ atmel_sha_authenc_free(ctx->auth);
+}
+
+static int atmel_aes_authenc_crypt(struct aead_request *req,
+ unsigned long mode)
+{
+ struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm);
+ u32 authsize = crypto_aead_authsize(tfm);
+ bool enc = (mode & AES_FLAGS_ENCRYPT);
+ struct atmel_aes_dev *dd;
+
+ /* Compute text length. */
+ if (!enc && req->cryptlen < authsize)
+ return -EINVAL;
+ rctx->textlen = req->cryptlen - (enc ? 0 : authsize);
+
+ /*
+ * Currently, empty messages are not supported yet:
+ * the SHA auto-padding can be used only on non-empty messages.
+ * Hence a special case needs to be implemented for empty message.
+ */
+ if (!rctx->textlen && !req->assoclen)
+ return -EINVAL;
+
+ rctx->base.mode = mode;
+ ctx->block_size = AES_BLOCK_SIZE;
+
+ dd = atmel_aes_find_dev(ctx);
+ if (!dd)
+ return -ENODEV;
+
+ return atmel_aes_handle_queue(dd, &req->base);
+}
+
+static int atmel_aes_authenc_cbc_aes_encrypt(struct aead_request *req)
+{
+ return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT);
+}
+
+static int atmel_aes_authenc_cbc_aes_decrypt(struct aead_request *req)
+{
+ return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC);
+}
+
+static struct aead_alg aes_authenc_algs[] = {
+{
+ .setkey = atmel_aes_authenc_setkey,
+ .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
+ .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
+ .init = atmel_aes_authenc_hmac_sha1_init_tfm,
+ .exit = atmel_aes_authenc_exit_tfm,
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA1_DIGEST_SIZE,
+
+ .base = {
+ .cra_name = "authenc(hmac(sha1),cbc(aes))",
+ .cra_driver_name = "atmel-authenc-hmac-sha1-cbc-aes",
+ .cra_priority = ATMEL_AES_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ },
+},
+{
+ .setkey = atmel_aes_authenc_setkey,
+ .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
+ .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
+ .init = atmel_aes_authenc_hmac_sha224_init_tfm,
+ .exit = atmel_aes_authenc_exit_tfm,
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA224_DIGEST_SIZE,
+
+ .base = {
+ .cra_name = "authenc(hmac(sha224),cbc(aes))",
+ .cra_driver_name = "atmel-authenc-hmac-sha224-cbc-aes",
+ .cra_priority = ATMEL_AES_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ },
+},
+{
+ .setkey = atmel_aes_authenc_setkey,
+ .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
+ .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
+ .init = atmel_aes_authenc_hmac_sha256_init_tfm,
+ .exit = atmel_aes_authenc_exit_tfm,
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA256_DIGEST_SIZE,
+
+ .base = {
+ .cra_name = "authenc(hmac(sha256),cbc(aes))",
+ .cra_driver_name = "atmel-authenc-hmac-sha256-cbc-aes",
+ .cra_priority = ATMEL_AES_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ },
+},
+{
+ .setkey = atmel_aes_authenc_setkey,
+ .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
+ .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
+ .init = atmel_aes_authenc_hmac_sha384_init_tfm,
+ .exit = atmel_aes_authenc_exit_tfm,
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA384_DIGEST_SIZE,
+
+ .base = {
+ .cra_name = "authenc(hmac(sha384),cbc(aes))",
+ .cra_driver_name = "atmel-authenc-hmac-sha384-cbc-aes",
+ .cra_priority = ATMEL_AES_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ },
+},
+{
+ .setkey = atmel_aes_authenc_setkey,
+ .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
+ .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
+ .init = atmel_aes_authenc_hmac_sha512_init_tfm,
+ .exit = atmel_aes_authenc_exit_tfm,
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA512_DIGEST_SIZE,
+
+ .base = {
+ .cra_name = "authenc(hmac(sha512),cbc(aes))",
+ .cra_driver_name = "atmel-authenc-hmac-sha512-cbc-aes",
+ .cra_priority = ATMEL_AES_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ },
+},
+};
+#endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */
/* Probe functions */
@@ -2040,6 +2453,12 @@ static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd)
{
int i;
+#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
+ if (dd->caps.has_authenc)
+ for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++)
+ crypto_unregister_aead(&aes_authenc_algs[i]);
+#endif
+
if (dd->caps.has_xts)
crypto_unregister_alg(&aes_xts_alg);
@@ -2081,8 +2500,25 @@ static int atmel_aes_register_algs(struct atmel_aes_dev *dd)
goto err_aes_xts_alg;
}
+#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
+ if (dd->caps.has_authenc) {
+ for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) {
+ err = crypto_register_aead(&aes_authenc_algs[i]);
+ if (err)
+ goto err_aes_authenc_alg;
+ }
+ }
+#endif
+
return 0;
+#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
+ /* i = ARRAY_SIZE(aes_authenc_algs); */
+err_aes_authenc_alg:
+ for (j = 0; j < i; j++)
+ crypto_unregister_aead(&aes_authenc_algs[j]);
+ crypto_unregister_alg(&aes_xts_alg);
+#endif
err_aes_xts_alg:
crypto_unregister_aead(&aes_gcm_alg);
err_aes_gcm_alg:
@@ -2103,6 +2539,7 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd)
dd->caps.has_ctr32 = 0;
dd->caps.has_gcm = 0;
dd->caps.has_xts = 0;
+ dd->caps.has_authenc = 0;
dd->caps.max_burst_size = 1;
/* keep only major version number */
@@ -2113,6 +2550,7 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd)
dd->caps.has_ctr32 = 1;
dd->caps.has_gcm = 1;
dd->caps.has_xts = 1;
+ dd->caps.has_authenc = 1;
dd->caps.max_burst_size = 4;
break;
case 0x200:
@@ -2271,6 +2709,13 @@ static int atmel_aes_probe(struct platform_device *pdev)
atmel_aes_get_cap(aes_dd);
+#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
+ if (aes_dd->caps.has_authenc && !atmel_sha_authenc_is_ready()) {
+ err = -EPROBE_DEFER;
+ goto iclk_unprepare;
+ }
+#endif
+
err = atmel_aes_buff_init(aes_dd);
if (err)
goto err_aes_buff;
@@ -2307,7 +2752,8 @@ static int atmel_aes_probe(struct platform_device *pdev)
tasklet_kill(&aes_dd->done_task);
tasklet_kill(&aes_dd->queue_task);
aes_dd_err:
- dev_err(dev, "initialization failed.\n");
+ if (err != -EPROBE_DEFER)
+ dev_err(dev, "initialization failed.\n");
return err;
}
diff --git a/drivers/crypto/atmel-authenc.h b/drivers/crypto/atmel-authenc.h
new file mode 100644
index 000000000000..2a60d1224143
--- /dev/null
+++ b/drivers/crypto/atmel-authenc.h
@@ -0,0 +1,64 @@
+/*
+ * API for Atmel Secure Protocol Layers Improved Performances (SPLIP)
+ *
+ * Copyright (C) 2016 Atmel Corporation
+ *
+ * Author: Cyrille Pitchen <cyrille.pitchen@atmel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * This driver is based on drivers/mtd/spi-nor/fsl-quadspi.c from Freescale.
+ */
+
+#ifndef __ATMEL_AUTHENC_H__
+#define __ATMEL_AUTHENC_H__
+
+#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
+
+#include <crypto/authenc.h>
+#include <crypto/hash.h>
+#include <crypto/sha.h>
+#include "atmel-sha-regs.h"
+
+struct atmel_aes_dev;
+typedef int (*atmel_aes_authenc_fn_t)(struct atmel_aes_dev *, int, bool);
+
+struct atmel_sha_authenc_ctx;
+
+bool atmel_sha_authenc_is_ready(void);
+unsigned int atmel_sha_authenc_get_reqsize(void);
+
+struct atmel_sha_authenc_ctx *atmel_sha_authenc_spawn(unsigned long mode);
+void atmel_sha_authenc_free(struct atmel_sha_authenc_ctx *auth);
+int atmel_sha_authenc_setkey(struct atmel_sha_authenc_ctx *auth,
+ const u8 *key, unsigned int keylen,
+ u32 *flags);
+
+int atmel_sha_authenc_schedule(struct ahash_request *req,
+ struct atmel_sha_authenc_ctx *auth,
+ atmel_aes_authenc_fn_t cb,
+ struct atmel_aes_dev *dd);
+int atmel_sha_authenc_init(struct ahash_request *req,
+ struct scatterlist *assoc, unsigned int assoclen,
+ unsigned int textlen,
+ atmel_aes_authenc_fn_t cb,
+ struct atmel_aes_dev *dd);
+int atmel_sha_authenc_final(struct ahash_request *req,
+ u32 *digest, unsigned int digestlen,
+ atmel_aes_authenc_fn_t cb,
+ struct atmel_aes_dev *dd);
+void atmel_sha_authenc_abort(struct ahash_request *req);
+
+#endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */
+
+#endif /* __ATMEL_AUTHENC_H__ */
diff --git a/drivers/crypto/atmel-sha-regs.h b/drivers/crypto/atmel-sha-regs.h
index 1b9f3d33079e..1b0eba4a2706 100644
--- a/drivers/crypto/atmel-sha-regs.h
+++ b/drivers/crypto/atmel-sha-regs.h
@@ -29,6 +29,20 @@
#define SHA_MR_HMAC (1 << 11)
#define SHA_MR_DUALBUFF (1 << 16)
+#define SHA_FLAGS_ALGO_MASK SHA_MR_ALGO_MASK
+#define SHA_FLAGS_SHA1 SHA_MR_ALGO_SHA1
+#define SHA_FLAGS_SHA256 SHA_MR_ALGO_SHA256
+#define SHA_FLAGS_SHA384 SHA_MR_ALGO_SHA384
+#define SHA_FLAGS_SHA512 SHA_MR_ALGO_SHA512
+#define SHA_FLAGS_SHA224 SHA_MR_ALGO_SHA224
+#define SHA_FLAGS_HMAC SHA_MR_HMAC
+#define SHA_FLAGS_HMAC_SHA1 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA1)
+#define SHA_FLAGS_HMAC_SHA256 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA256)
+#define SHA_FLAGS_HMAC_SHA384 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA384)
+#define SHA_FLAGS_HMAC_SHA512 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA512)
+#define SHA_FLAGS_HMAC_SHA224 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA224)
+#define SHA_FLAGS_MODE_MASK (SHA_FLAGS_HMAC | SHA_FLAGS_ALGO_MASK)
+
#define SHA_IER 0x10
#define SHA_IDR 0x14
#define SHA_IMR 0x18
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index 78c3c02e4483..cc5294dbead4 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -41,6 +41,7 @@
#include <crypto/internal/hash.h>
#include <linux/platform_data/crypto-atmel.h>
#include "atmel-sha-regs.h"
+#include "atmel-authenc.h"
/* SHA flags */
#define SHA_FLAGS_BUSY BIT(0)
@@ -52,19 +53,6 @@
#define SHA_FLAGS_DMA_READY BIT(6)
/* bits[11:8] are reserved. */
-#define SHA_FLAGS_ALGO_MASK SHA_MR_ALGO_MASK
-#define SHA_FLAGS_SHA1 SHA_MR_ALGO_SHA1
-#define SHA_FLAGS_SHA256 SHA_MR_ALGO_SHA256
-#define SHA_FLAGS_SHA384 SHA_MR_ALGO_SHA384
-#define SHA_FLAGS_SHA512 SHA_MR_ALGO_SHA512
-#define SHA_FLAGS_SHA224 SHA_MR_ALGO_SHA224
-#define SHA_FLAGS_HMAC SHA_MR_HMAC
-#define SHA_FLAGS_HMAC_SHA1 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA1)
-#define SHA_FLAGS_HMAC_SHA256 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA256)
-#define SHA_FLAGS_HMAC_SHA384 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA384)
-#define SHA_FLAGS_HMAC_SHA512 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA512)
-#define SHA_FLAGS_HMAC_SHA224 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA224)
-#define SHA_FLAGS_MODE_MASK (SHA_FLAGS_HMAC | SHA_FLAGS_ALGO_MASK)
#define SHA_FLAGS_FINUP BIT(16)
#define SHA_FLAGS_SG BIT(17)
@@ -156,6 +144,7 @@ struct atmel_sha_dev {
struct crypto_queue queue;
struct ahash_request *req;
bool is_async;
+ bool force_complete;
atmel_sha_fn_t resume;
atmel_sha_fn_t cpu_transfer_complete;
@@ -198,7 +187,7 @@ static inline int atmel_sha_complete(struct atmel_sha_dev *dd, int err)
clk_disable(dd->iclk);
- if (dd->is_async && req->base.complete)
+ if ((dd->is_async || dd->force_complete) && req->base.complete)
req->base.complete(&req->base, err);
/* handle new request */
@@ -992,6 +981,7 @@ static int atmel_sha_handle_queue(struct atmel_sha_dev *dd,
dd->req = ahash_request_cast(async_req);
start_async = (dd->req != req);
dd->is_async = start_async;
+ dd->force_complete = false;
/* WARNING: ctx->start() MAY change dd->is_async. */
err = ctx->start(dd);
@@ -2100,6 +2090,332 @@ static struct ahash_alg sha_hmac_algs[] = {
},
};
+#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
+/* authenc functions */
+
+static int atmel_sha_authenc_init2(struct atmel_sha_dev *dd);
+static int atmel_sha_authenc_init_done(struct atmel_sha_dev *dd);
+static int atmel_sha_authenc_final_done(struct atmel_sha_dev *dd);
+
+
+struct atmel_sha_authenc_ctx {
+ struct crypto_ahash *tfm;
+};
+
+struct atmel_sha_authenc_reqctx {
+ struct atmel_sha_reqctx base;
+
+ atmel_aes_authenc_fn_t cb;
+ struct atmel_aes_dev *aes_dev;
+
+ /* _init() parameters. */
+ struct scatterlist *assoc;
+ u32 assoclen;
+ u32 textlen;
+
+ /* _final() parameters. */
+ u32 *digest;
+ unsigned int digestlen;
+};
+
+static void atmel_sha_authenc_complete(struct crypto_async_request *areq,
+ int err)
+{
+ struct ahash_request *req = areq->data;
+ struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
+
+ authctx->cb(authctx->aes_dev, err, authctx->base.dd->is_async);
+}
+
+static int atmel_sha_authenc_start(struct atmel_sha_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
+ int err;
+
+ /*
+ * Force atmel_sha_complete() to call req->base.complete(), ie
+ * atmel_sha_authenc_complete(), which in turn calls authctx->cb().
+ */
+ dd->force_complete = true;
+
+ err = atmel_sha_hw_init(dd);
+ return authctx->cb(authctx->aes_dev, err, dd->is_async);
+}
+
+bool atmel_sha_authenc_is_ready(void)
+{
+ struct atmel_sha_ctx dummy;
+
+ dummy.dd = NULL;
+ return (atmel_sha_find_dev(&dummy) != NULL);
+}
+EXPORT_SYMBOL_GPL(atmel_sha_authenc_is_ready);
+
+unsigned int atmel_sha_authenc_get_reqsize(void)
+{
+ return sizeof(struct atmel_sha_authenc_reqctx);
+}
+EXPORT_SYMBOL_GPL(atmel_sha_authenc_get_reqsize);
+
+struct atmel_sha_authenc_ctx *atmel_sha_authenc_spawn(unsigned long mode)
+{
+ struct atmel_sha_authenc_ctx *auth;
+ struct crypto_ahash *tfm;
+ struct atmel_sha_ctx *tctx;
+ const char *name;
+ int err = -EINVAL;
+
+ switch (mode & SHA_FLAGS_MODE_MASK) {
+ case SHA_FLAGS_HMAC_SHA1:
+ name = "atmel-hmac-sha1";
+ break;
+
+ case SHA_FLAGS_HMAC_SHA224:
+ name = "atmel-hmac-sha224";
+ break;
+
+ case SHA_FLAGS_HMAC_SHA256:
+ name = "atmel-hmac-sha256";
+ break;
+
+ case SHA_FLAGS_HMAC_SHA384:
+ name = "atmel-hmac-sha384";
+ break;
+
+ case SHA_FLAGS_HMAC_SHA512:
+ name = "atmel-hmac-sha512";
+ break;
+
+ default:
+ goto error;
+ }
+
+ tfm = crypto_alloc_ahash(name,
+ CRYPTO_ALG_TYPE_AHASH,
+ CRYPTO_ALG_TYPE_AHASH_MASK);
+ if (IS_ERR(tfm)) {
+ err = PTR_ERR(tfm);
+ goto error;
+ }
+ tctx = crypto_ahash_ctx(tfm);
+ tctx->start = atmel_sha_authenc_start;
+ tctx->flags = mode;
+
+ auth = kzalloc(sizeof(*auth), GFP_KERNEL);
+ if (!auth) {
+ err = -ENOMEM;
+ goto err_free_ahash;
+ }
+ auth->tfm = tfm;
+
+ return auth;
+
+err_free_ahash:
+ crypto_free_ahash(tfm);
+error:
+ return ERR_PTR(err);
+}
+EXPORT_SYMBOL_GPL(atmel_sha_authenc_spawn);
+
+void atmel_sha_authenc_free(struct atmel_sha_authenc_ctx *auth)
+{
+ if (auth)
+ crypto_free_ahash(auth->tfm);
+ kfree(auth);
+}
+EXPORT_SYMBOL_GPL(atmel_sha_authenc_free);
+
+int atmel_sha_authenc_setkey(struct atmel_sha_authenc_ctx *auth,
+ const u8 *key, unsigned int keylen,
+ u32 *flags)
+{
+ struct crypto_ahash *tfm = auth->tfm;
+ int err;
+
+ crypto_ahash_clear_flags(tfm, CRYPTO_TFM_REQ_MASK);
+ crypto_ahash_set_flags(tfm, *flags & CRYPTO_TFM_REQ_MASK);
+ err = crypto_ahash_setkey(tfm, key, keylen);
+ *flags = crypto_ahash_get_flags(tfm);
+
+ return err;
+}
+EXPORT_SYMBOL_GPL(atmel_sha_authenc_setkey);
+
+int atmel_sha_authenc_schedule(struct ahash_request *req,
+ struct atmel_sha_authenc_ctx *auth,
+ atmel_aes_authenc_fn_t cb,
+ struct atmel_aes_dev *aes_dev)
+{
+ struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
+ struct atmel_sha_reqctx *ctx = &authctx->base;
+ struct crypto_ahash *tfm = auth->tfm;
+ struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct atmel_sha_dev *dd;
+
+ /* Reset request context (MUST be done first). */
+ memset(authctx, 0, sizeof(*authctx));
+
+ /* Get SHA device. */
+ dd = atmel_sha_find_dev(tctx);
+ if (!dd)
+ return cb(aes_dev, -ENODEV, false);
+
+ /* Init request context. */
+ ctx->dd = dd;
+ ctx->buflen = SHA_BUFFER_LEN;
+ authctx->cb = cb;
+ authctx->aes_dev = aes_dev;
+ ahash_request_set_tfm(req, tfm);
+ ahash_request_set_callback(req, 0, atmel_sha_authenc_complete, req);
+
+ return atmel_sha_handle_queue(dd, req);
+}
+EXPORT_SYMBOL_GPL(atmel_sha_authenc_schedule);
+
+int atmel_sha_authenc_init(struct ahash_request *req,
+ struct scatterlist *assoc, unsigned int assoclen,
+ unsigned int textlen,
+ atmel_aes_authenc_fn_t cb,
+ struct atmel_aes_dev *aes_dev)
+{
+ struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
+ struct atmel_sha_reqctx *ctx = &authctx->base;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
+ struct atmel_sha_dev *dd = ctx->dd;
+
+ if (unlikely(!IS_ALIGNED(assoclen, sizeof(u32))))
+ return atmel_sha_complete(dd, -EINVAL);
+
+ authctx->cb = cb;
+ authctx->aes_dev = aes_dev;
+ authctx->assoc = assoc;
+ authctx->assoclen = assoclen;
+ authctx->textlen = textlen;
+
+ ctx->flags = hmac->base.flags;
+ return atmel_sha_hmac_setup(dd, atmel_sha_authenc_init2);
+}
+EXPORT_SYMBOL_GPL(atmel_sha_authenc_init);
+
+static int atmel_sha_authenc_init2(struct atmel_sha_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
+ struct atmel_sha_reqctx *ctx = &authctx->base;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
+ size_t hs = ctx->hash_size;
+ size_t i, num_words = hs / sizeof(u32);
+ u32 mr, msg_size;
+
+ atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV);
+ for (i = 0; i < num_words; ++i)
+ atmel_sha_write(dd, SHA_REG_DIN(i), hmac->ipad[i]);
+
+ atmel_sha_write(dd, SHA_CR, SHA_CR_WUIEHV);
+ for (i = 0; i < num_words; ++i)
+ atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]);
+
+ mr = (SHA_MR_MODE_IDATAR0 |
+ SHA_MR_HMAC |
+ SHA_MR_DUALBUFF);
+ mr |= ctx->flags & SHA_FLAGS_ALGO_MASK;
+ atmel_sha_write(dd, SHA_MR, mr);
+
+ msg_size = authctx->assoclen + authctx->textlen;
+ atmel_sha_write(dd, SHA_MSR, msg_size);
+ atmel_sha_write(dd, SHA_BCR, msg_size);
+
+ atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);
+
+ /* Process assoc data. */
+ return atmel_sha_cpu_start(dd, authctx->assoc, authctx->assoclen,
+ true, false,
+ atmel_sha_authenc_init_done);
+}
+
+static int atmel_sha_authenc_init_done(struct atmel_sha_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
+
+ return authctx->cb(authctx->aes_dev, 0, dd->is_async);
+}
+
+int atmel_sha_authenc_final(struct ahash_request *req,
+ u32 *digest, unsigned int digestlen,
+ atmel_aes_authenc_fn_t cb,
+ struct atmel_aes_dev *aes_dev)
+{
+ struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
+ struct atmel_sha_reqctx *ctx = &authctx->base;
+ struct atmel_sha_dev *dd = ctx->dd;
+
+ switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {
+ case SHA_FLAGS_SHA1:
+ authctx->digestlen = SHA1_DIGEST_SIZE;
+ break;
+
+ case SHA_FLAGS_SHA224:
+ authctx->digestlen = SHA224_DIGEST_SIZE;
+ break;
+
+ case SHA_FLAGS_SHA256:
+ authctx->digestlen = SHA256_DIGEST_SIZE;
+ break;
+
+ case SHA_FLAGS_SHA384:
+ authctx->digestlen = SHA384_DIGEST_SIZE;
+ break;
+
+ case SHA_FLAGS_SHA512:
+ authctx->digestlen = SHA512_DIGEST_SIZE;
+ break;
+
+ default:
+ return atmel_sha_complete(dd, -EINVAL);
+ }
+ if (authctx->digestlen > digestlen)
+ authctx->digestlen = digestlen;
+
+ authctx->cb = cb;
+ authctx->aes_dev = aes_dev;
+ authctx->digest = digest;
+ return atmel_sha_wait_for_data_ready(dd,
+ atmel_sha_authenc_final_done);
+}
+EXPORT_SYMBOL_GPL(atmel_sha_authenc_final);
+
+static int atmel_sha_authenc_final_done(struct atmel_sha_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
+ size_t i, num_words = authctx->digestlen / sizeof(u32);
+
+ for (i = 0; i < num_words; ++i)
+ authctx->digest[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));
+
+ return atmel_sha_complete(dd, 0);
+}
+
+void atmel_sha_authenc_abort(struct ahash_request *req)
+{
+ struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
+ struct atmel_sha_reqctx *ctx = &authctx->base;
+ struct atmel_sha_dev *dd = ctx->dd;
+
+ /* Prevent atmel_sha_complete() from calling req->base.complete(). */
+ dd->is_async = false;
+ dd->force_complete = false;
+ (void)atmel_sha_complete(dd, 0);
+}
+EXPORT_SYMBOL_GPL(atmel_sha_authenc_abort);
+
+#endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */
+
+
static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd)
{
int i;
--
2.7.4
^ permalink raw reply related
* [PATCH v3 12/12] crypto: atmel-sha: add verbose debug facilities to print hw register names
From: Cyrille Pitchen @ 2017-01-26 16:07 UTC (permalink / raw)
To: herbert, davem, nicolas.ferre
Cc: smueller, linux-crypto, linux-kernel, linux-arm-kernel,
Cyrille Pitchen
In-Reply-To: <cover.1485443478.git.cyrille.pitchen@atmel.com>
When VERBOSE_DEBUG is defined and SHA_FLAGS_DUMP_REG flag is set in
dd->flags, this patch prints the register names and values when performing
IO accesses.
Signed-off-by: Cyrille Pitchen <cyrille.pitchen@atmel.com>
---
drivers/crypto/atmel-sha.c | 110 ++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 108 insertions(+), 2 deletions(-)
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index cc5294dbead4..22d0c0c118da 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -51,6 +51,7 @@
#define SHA_FLAGS_INIT BIT(4)
#define SHA_FLAGS_CPU BIT(5)
#define SHA_FLAGS_DMA_READY BIT(6)
+#define SHA_FLAGS_DUMP_REG BIT(7)
/* bits[11:8] are reserved. */
@@ -167,14 +168,118 @@ static struct atmel_sha_drv atmel_sha = {
.lock = __SPIN_LOCK_UNLOCKED(atmel_sha.lock),
};
+#ifdef VERBOSE_DEBUG
+static const char *atmel_sha_reg_name(u32 offset, char *tmp, size_t sz, bool wr)
+{
+ switch (offset) {
+ case SHA_CR:
+ return "CR";
+
+ case SHA_MR:
+ return "MR";
+
+ case SHA_IER:
+ return "IER";
+
+ case SHA_IDR:
+ return "IDR";
+
+ case SHA_IMR:
+ return "IMR";
+
+ case SHA_ISR:
+ return "ISR";
+
+ case SHA_MSR:
+ return "MSR";
+
+ case SHA_BCR:
+ return "BCR";
+
+ case SHA_REG_DIN(0):
+ case SHA_REG_DIN(1):
+ case SHA_REG_DIN(2):
+ case SHA_REG_DIN(3):
+ case SHA_REG_DIN(4):
+ case SHA_REG_DIN(5):
+ case SHA_REG_DIN(6):
+ case SHA_REG_DIN(7):
+ case SHA_REG_DIN(8):
+ case SHA_REG_DIN(9):
+ case SHA_REG_DIN(10):
+ case SHA_REG_DIN(11):
+ case SHA_REG_DIN(12):
+ case SHA_REG_DIN(13):
+ case SHA_REG_DIN(14):
+ case SHA_REG_DIN(15):
+ snprintf(tmp, sz, "IDATAR[%u]", (offset - SHA_REG_DIN(0)) >> 2);
+ break;
+
+ case SHA_REG_DIGEST(0):
+ case SHA_REG_DIGEST(1):
+ case SHA_REG_DIGEST(2):
+ case SHA_REG_DIGEST(3):
+ case SHA_REG_DIGEST(4):
+ case SHA_REG_DIGEST(5):
+ case SHA_REG_DIGEST(6):
+ case SHA_REG_DIGEST(7):
+ case SHA_REG_DIGEST(8):
+ case SHA_REG_DIGEST(9):
+ case SHA_REG_DIGEST(10):
+ case SHA_REG_DIGEST(11):
+ case SHA_REG_DIGEST(12):
+ case SHA_REG_DIGEST(13):
+ case SHA_REG_DIGEST(14):
+ case SHA_REG_DIGEST(15):
+ if (wr)
+ snprintf(tmp, sz, "IDATAR[%u]",
+ 16u + ((offset - SHA_REG_DIGEST(0)) >> 2));
+ else
+ snprintf(tmp, sz, "ODATAR[%u]",
+ (offset - SHA_REG_DIGEST(0)) >> 2);
+ break;
+
+ case SHA_HW_VERSION:
+ return "HWVER";
+
+ default:
+ snprintf(tmp, sz, "0x%02x", offset);
+ break;
+ }
+
+ return tmp;
+}
+
+#endif /* VERBOSE_DEBUG */
+
static inline u32 atmel_sha_read(struct atmel_sha_dev *dd, u32 offset)
{
- return readl_relaxed(dd->io_base + offset);
+ u32 value = readl_relaxed(dd->io_base + offset);
+
+#ifdef VERBOSE_DEBUG
+ if (dd->flags & SHA_FLAGS_DUMP_REG) {
+ char tmp[16];
+
+ dev_vdbg(dd->dev, "read 0x%08x from %s\n", value,
+ atmel_sha_reg_name(offset, tmp, sizeof(tmp), false));
+ }
+#endif /* VERBOSE_DEBUG */
+
+ return value;
}
static inline void atmel_sha_write(struct atmel_sha_dev *dd,
u32 offset, u32 value)
{
+#ifdef VERBOSE_DEBUG
+ if (dd->flags & SHA_FLAGS_DUMP_REG) {
+ char tmp[16];
+
+ dev_vdbg(dd->dev, "write 0x%08x into %s\n", value,
+ atmel_sha_reg_name(offset, tmp, sizeof(tmp), true));
+ }
+#endif /* VERBOSE_DEBUG */
+
writel_relaxed(value, dd->io_base + offset);
}
@@ -183,7 +288,8 @@ static inline int atmel_sha_complete(struct atmel_sha_dev *dd, int err)
struct ahash_request *req = dd->req;
dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU |
- SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY);
+ SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY |
+ SHA_FLAGS_DUMP_REG);
clk_disable(dd->iclk);
--
2.7.4
^ permalink raw reply related
* [PATCH v3 00/12] crypto: atmel-authenc: add support to authenc(hmac(shaX),Y(aes)) modes
From: Cyrille Pitchen @ 2017-01-26 16:07 UTC (permalink / raw)
To: herbert, davem, nicolas.ferre
Cc: smueller, linux-crypto, linux-kernel, linux-arm-kernel,
Cyrille Pitchen
Hi all,
this series of patches has been based and tested on next-20170125 with
CRYPTO_MANAGER_DISABLED_TESTS not set.
The series adds support to the hmac(shaX) algorithms first, then combines
both the Atmel SHA and AES hardware accelerators to implement
authenc(hmac(shaX),Y(aes)) algorithms as used by IPSEC/SSL connections.
It has also been tested with strongswan + xl2tpd to create an IPSEC+L2TP
(transport mode) VPN and strongswan only (tunnel mode) for an IPSEC VPN.
Then iperf was used to measure the bandwidth improvement in tunnel mode:
drivers AES SHA SPLIP iperf half-duplex
Mbit/s
authenc(hmac(sha1-generic),cbc(aes)) SW SW N/A 27.7
authenc(hmac(sha1-generic),atmel-cbc-aes) HW SW N/A 30.2 (mainline)
authenc(atmel-hmac-sha1,atmel-cbc-aes) HW HW no 29.1
atmel-authenc-hmac-sha1-cbc-aes HW HW yes 38.8
SPLIP: Secure Protocol Layers Improved Performances (AES+SHA combined).
Some patches of this series are purely transitional: I've split the
modifications into many patches to ease the review.
Best regards,
Cyrille
ChangeLog:
v2 -> v3:
- add calls to memzero_explicit(&keys, ...) before exiting from
atmel_aes_authenc_setkey().
- add missing comparison test between req->cryptlen and authsize values
in atmel_aes_authenc_crypt().
- remove the atmel_aes_authenc_copy_assoc() function: I have tested with
a strongswan IPSec ESP Tunnel Mode connection and it still works even if
this driver no longer copies the AAD. FWI, I have also tested with the
crypto/authenc.c driver after having removed the
crypto_authenc_copy_assoc() function and it worked as well. However I
didn't check whether I was in the special case req->src == req->dst.
v1 -> v2:
- add missing drivers/crypto/atmel-authenc.h file in patch 11.
Cyrille Pitchen (12):
crypto: atmel-sha: create function to get an Atmel SHA device
crypto: atmel-sha: update request queue management to make it more
generic
crypto: atmel-sha: make atmel_sha_done_task more generic
crypto: atmel-sha: redefine SHA_FLAGS_SHA* flags to match
SHA_MR_ALGO_SHA*
crypto: atmel-sha: add atmel_sha_wait_for_data_ready()
crypto: atmel-sha: add SHA_MR_MODE_IDATAR0
crypto: atmel-sha: add atmel_sha_cpu_start()
crypto: atmel-sha: add simple DMA transfers
crypto: atmel-sha: add support to hmac(shaX)
crypto: atmel-aes: fix atmel_aes_handle_queue()
crypto: atmel-authenc: add support to authenc(hmac(shaX),Y(aes)) modes
crypto: atmel-sha: add verbose debug facilities to print hw register
names
drivers/crypto/Kconfig | 12 +
drivers/crypto/atmel-aes-regs.h | 16 +
drivers/crypto/atmel-aes.c | 455 ++++++++++++-
drivers/crypto/atmel-authenc.h | 64 ++
drivers/crypto/atmel-sha-regs.h | 20 +
drivers/crypto/atmel-sha.c | 1438 +++++++++++++++++++++++++++++++++++++--
6 files changed, 1937 insertions(+), 68 deletions(-)
create mode 100644 drivers/crypto/atmel-authenc.h
--
2.7.4
^ permalink raw reply
* [PATCH v3 05/12] crypto: atmel-sha: add atmel_sha_wait_for_data_ready()
From: Cyrille Pitchen @ 2017-01-26 16:07 UTC (permalink / raw)
To: herbert, davem, nicolas.ferre
Cc: smueller, linux-crypto, linux-kernel, linux-arm-kernel,
Cyrille Pitchen
In-Reply-To: <cover.1485443478.git.cyrille.pitchen@atmel.com>
This patch simply defines a helper function to test the 'Data Ready' flag
of the Status Register. It also gives a chance for the crypto request to
be processed synchronously if this 'Data Ready' flag is already set when
polling the Status Register. Indeed, running synchronously avoid the
latency of the 'Data Ready' interrupt.
When the 'Data Ready' flag has not been set yet, we enable the associated
interrupt and resume processing the crypto request asynchronously from the
'done' task just as before.
Signed-off-by: Cyrille Pitchen <cyrille.pitchen@atmel.com>
---
drivers/crypto/atmel-sha.c | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index b29a4e5bc404..be0d72cf4352 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -434,6 +434,19 @@ static void atmel_sha_write_ctrl(struct atmel_sha_dev *dd, int dma)
atmel_sha_write(dd, SHA_MR, valmr);
}
+static inline int atmel_sha_wait_for_data_ready(struct atmel_sha_dev *dd,
+ atmel_sha_fn_t resume)
+{
+ u32 isr = atmel_sha_read(dd, SHA_ISR);
+
+ if (unlikely(isr & SHA_INT_DATARDY))
+ return resume(dd);
+
+ dd->resume = resume;
+ atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);
+ return -EINPROGRESS;
+}
+
static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf,
size_t length, int final)
{
--
2.7.4
^ permalink raw reply related
* [PATCH 0/4] crypto: time invariant AES for CCM (and GCM/CTR)
From: Ard Biesheuvel @ 2017-01-26 17:17 UTC (permalink / raw)
To: linux-crypto, linux-arm-kernel; +Cc: herbert, Ard Biesheuvel
This series is primarily directed at improving the performance and security
of CCM on the Rasperry Pi 3. This involves splitting the MAC handling of
CCM into a separate driver so that we can efficiently replace it by something
else using the ordinary algo resolution machinery.
Patch #1 adds some testcases for cbcmac(aes), which will be introduced later.
Patch #2 replaces the open coded CBC MAC hashing routines in the CCM driver
with calls to a cbcmac() hash, and implements a template for producing such
transforms. This eliminates all the fuzzy scatterwalk code as well.
Patch #3 implements cbcmac(aes) using NEON on arm64
Patch #4 is an RFC patch that implements ctr(aes) and cbcmac(aes) in a way
that is intended to eliminate observeable data dependent latencies in AES
processing, by replacing the usual 16 KB of lookup tables with a single
Sbox that is prefetched before processing each block. It is 50% slower than
generic AES, but this may be acceptable in many cases.
Ard Biesheuvel (4):
crypto: testmgr - add test cases for cbcmac(aes)
crypto: ccm - switch to separate cbcmac driver
crypto: arm64/aes - add NEON and Crypto Extension CBC-MAC driver
crypto: aes - add generic time invariant AES for CTR/CCM/GCM
arch/arm64/crypto/aes-glue.c | 102 ++++++
arch/arm64/crypto/aes-modes.S | 19 +
crypto/Kconfig | 15 +
crypto/Makefile | 1 +
crypto/aes_ti.c | 314 ++++++++++++++++
crypto/ccm.c | 373 +++++++++++++-------
crypto/testmgr.c | 7 +
crypto/testmgr.h | 58 +++
8 files changed, 753 insertions(+), 136 deletions(-)
create mode 100644 crypto/aes_ti.c
--
2.7.4
^ permalink raw reply
* [PATCH 1/4] crypto: testmgr - add test cases for cbcmac(aes)
From: Ard Biesheuvel @ 2017-01-26 17:17 UTC (permalink / raw)
To: linux-crypto, linux-arm-kernel; +Cc: herbert, Ard Biesheuvel
In-Reply-To: <1485451063-11822-1-git-send-email-ard.biesheuvel@linaro.org>
In preparation of splitting off the CBC-MAC transform in the CCM
driver into a separate algorithm, define some test cases for the
AES incarnation of cbcmac.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
crypto/testmgr.c | 7 +++
crypto/testmgr.h | 58 ++++++++++++++++++++
2 files changed, 65 insertions(+)
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 98eb09782db8..f9c378af3907 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -2514,6 +2514,13 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}
}, {
+ .alg = "cbcmac(aes)",
+ .fips_allowed = 1,
+ .test = alg_test_hash,
+ .suite = {
+ .hash = __VECS(aes_cbcmac_tv_template)
+ }
+ }, {
.alg = "ccm(aes)",
.test = alg_test_aead,
.fips_allowed = 1,
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 64595f067d72..ed6b09978611 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -3413,6 +3413,64 @@ static struct hash_testvec aes_cmac128_tv_template[] = {
}
};
+static struct hash_testvec aes_cbcmac_tv_template[] = {
+ {
+ .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6"
+ "\xab\xf7\x15\x88\x09\xcf\x4f\x3c",
+ .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+ "\xe9\x3d\x7e\x11\x73\x93\x17\x2a",
+ .digest = "\x3a\xd7\x7b\xb4\x0d\x7a\x36\x60"
+ "\xa8\x9e\xca\xf3\x24\x66\xef\x97",
+ .psize = 16,
+ .ksize = 16,
+ }, {
+ .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6"
+ "\xab\xf7\x15\x88\x09\xcf\x4f\x3c",
+ .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+ "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+ "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+ "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+ "\x30",
+ .digest = "\x9d\x0d\xd0\x63\xfb\xcb\x24\x43"
+ "\xf8\xf2\x76\x03\xac\x39\xb0\x9d",
+ .psize = 33,
+ .ksize = 16,
+ }, {
+ .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6"
+ "\xab\xf7\x15\x88\x09\xcf\x4f\x3c",
+ .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+ "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+ "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+ "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+ "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+ "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+ "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+ "\xad\x2b\x41\x7b\xe6\x6c\x37",
+ .digest = "\xc0\x71\x73\xb8\xa0\x2c\x11\x7c"
+ "\xaf\xdc\xb2\xf8\x89\x32\xa3\x3a",
+ .psize = 63,
+ .ksize = 16,
+ }, {
+ .key = "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
+ "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
+ "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
+ "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
+ .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+ "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+ "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+ "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+ "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+ "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+ "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+ "\xad\x2b\x41\x7b\xe6\x6c\x37\x10"
+ "\x1c",
+ .digest = "\x6a\x4e\xdb\x21\x47\x51\xdf\x4f"
+ "\xa8\x4d\x4c\x10\x3b\x72\x7d\xd6",
+ .psize = 65,
+ .ksize = 32,
+ }
+};
+
static struct hash_testvec des3_ede_cmac64_tv_template[] = {
/*
* From NIST Special Publication 800-38B, Three Key TDEA
--
2.7.4
^ permalink raw reply related
* [PATCH 2/4] crypto: ccm - switch to separate cbcmac driver
From: Ard Biesheuvel @ 2017-01-26 17:17 UTC (permalink / raw)
To: linux-crypto, linux-arm-kernel; +Cc: herbert, Ard Biesheuvel
In-Reply-To: <1485451063-11822-1-git-send-email-ard.biesheuvel@linaro.org>
Update the generic CCM driver to defer CBC-MAC processing to a
dedicated CBC-MAC ahash transform rather than open coding this
transform (and much of the associated scatterwalk plumbing) in
the CCM driver itself.
This cleans up the code considerably, but more importantly, it allows
the use of alternative CBC-MAC implementations that don't suffer from
performance degradation due to significant setup time (e.g., the NEON
based AES code needs to load the entire S-box into SIMD registers, which
cannot be amortized over the entire input when using the AES cipher
directly)
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
crypto/Kconfig | 1 +
crypto/ccm.c | 373 +++++++++++++-------
2 files changed, 238 insertions(+), 136 deletions(-)
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 160f08e721cc..e8269d1b0282 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -263,6 +263,7 @@ comment "Authenticated Encryption with Associated Data"
config CRYPTO_CCM
tristate "CCM support"
select CRYPTO_CTR
+ select CRYPTO_HASH
select CRYPTO_AEAD
help
Support for Counter with CBC MAC. Required for IPsec.
diff --git a/crypto/ccm.c b/crypto/ccm.c
index 26b924d1e582..635f11fc52e7 100644
--- a/crypto/ccm.c
+++ b/crypto/ccm.c
@@ -11,6 +11,7 @@
*/
#include <crypto/internal/aead.h>
+#include <crypto/internal/hash.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <linux/err.h>
@@ -23,11 +24,11 @@
struct ccm_instance_ctx {
struct crypto_skcipher_spawn ctr;
- struct crypto_spawn cipher;
+ struct crypto_ahash_spawn mac;
};
struct crypto_ccm_ctx {
- struct crypto_cipher *cipher;
+ struct crypto_ahash *mac;
struct crypto_skcipher *ctr;
};
@@ -44,7 +45,6 @@ struct crypto_rfc4309_req_ctx {
struct crypto_ccm_req_priv_ctx {
u8 odata[16];
- u8 idata[16];
u8 auth_tag[16];
u32 ilen;
u32 flags;
@@ -53,6 +53,15 @@ struct crypto_ccm_req_priv_ctx {
struct skcipher_request skreq;
};
+struct cbcmac_tfm_ctx {
+ struct crypto_cipher *child;
+};
+
+struct cbcmac_desc_ctx {
+ unsigned int len;
+ u8 dg[];
+};
+
static inline struct crypto_ccm_req_priv_ctx *crypto_ccm_reqctx(
struct aead_request *req)
{
@@ -84,7 +93,7 @@ static int crypto_ccm_setkey(struct crypto_aead *aead, const u8 *key,
{
struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
struct crypto_skcipher *ctr = ctx->ctr;
- struct crypto_cipher *tfm = ctx->cipher;
+ struct crypto_ahash *mac = ctx->mac;
int err = 0;
crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
@@ -96,11 +105,11 @@ static int crypto_ccm_setkey(struct crypto_aead *aead, const u8 *key,
if (err)
goto out;
- crypto_cipher_clear_flags(tfm, CRYPTO_TFM_REQ_MASK);
- crypto_cipher_set_flags(tfm, crypto_aead_get_flags(aead) &
+ crypto_ahash_clear_flags(mac, CRYPTO_TFM_REQ_MASK);
+ crypto_ahash_set_flags(mac, crypto_aead_get_flags(aead) &
CRYPTO_TFM_REQ_MASK);
- err = crypto_cipher_setkey(tfm, key, keylen);
- crypto_aead_set_flags(aead, crypto_cipher_get_flags(tfm) &
+ err = crypto_ahash_setkey(mac, key, keylen);
+ crypto_aead_set_flags(aead, crypto_ahash_get_flags(mac) &
CRYPTO_TFM_RES_MASK);
out:
@@ -167,119 +176,59 @@ static int format_adata(u8 *adata, unsigned int a)
return len;
}
-static void compute_mac(struct crypto_cipher *tfm, u8 *data, int n,
- struct crypto_ccm_req_priv_ctx *pctx)
-{
- unsigned int bs = 16;
- u8 *odata = pctx->odata;
- u8 *idata = pctx->idata;
- int datalen, getlen;
-
- datalen = n;
-
- /* first time in here, block may be partially filled. */
- getlen = bs - pctx->ilen;
- if (datalen >= getlen) {
- memcpy(idata + pctx->ilen, data, getlen);
- crypto_xor(odata, idata, bs);
- crypto_cipher_encrypt_one(tfm, odata, odata);
- datalen -= getlen;
- data += getlen;
- pctx->ilen = 0;
- }
-
- /* now encrypt rest of data */
- while (datalen >= bs) {
- crypto_xor(odata, data, bs);
- crypto_cipher_encrypt_one(tfm, odata, odata);
-
- datalen -= bs;
- data += bs;
- }
-
- /* check and see if there's leftover data that wasn't
- * enough to fill a block.
- */
- if (datalen) {
- memcpy(idata + pctx->ilen, data, datalen);
- pctx->ilen += datalen;
- }
-}
-
-static void get_data_to_compute(struct crypto_cipher *tfm,
- struct crypto_ccm_req_priv_ctx *pctx,
- struct scatterlist *sg, unsigned int len)
-{
- struct scatter_walk walk;
- u8 *data_src;
- int n;
-
- scatterwalk_start(&walk, sg);
-
- while (len) {
- n = scatterwalk_clamp(&walk, len);
- if (!n) {
- scatterwalk_start(&walk, sg_next(walk.sg));
- n = scatterwalk_clamp(&walk, len);
- }
- data_src = scatterwalk_map(&walk);
-
- compute_mac(tfm, data_src, n, pctx);
- len -= n;
-
- scatterwalk_unmap(data_src);
- scatterwalk_advance(&walk, n);
- scatterwalk_done(&walk, 0, len);
- if (len)
- crypto_yield(pctx->flags);
- }
-
- /* any leftover needs padding and then encrypted */
- if (pctx->ilen) {
- int padlen;
- u8 *odata = pctx->odata;
- u8 *idata = pctx->idata;
-
- padlen = 16 - pctx->ilen;
- memset(idata + pctx->ilen, 0, padlen);
- crypto_xor(odata, idata, 16);
- crypto_cipher_encrypt_one(tfm, odata, odata);
- pctx->ilen = 0;
- }
-}
-
static int crypto_ccm_auth(struct aead_request *req, struct scatterlist *plain,
unsigned int cryptlen)
{
+ struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
- struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
- struct crypto_cipher *cipher = ctx->cipher;
+ AHASH_REQUEST_ON_STACK(ahreq, ctx->mac);
unsigned int assoclen = req->assoclen;
- u8 *odata = pctx->odata;
- u8 *idata = pctx->idata;
- int err;
+ struct scatterlist sg[3];
+ u8 odata[16];
+ u8 idata[16];
+ int ilen, err;
/* format control data for input */
err = format_input(odata, req, cryptlen);
if (err)
goto out;
- /* encrypt first block to use as start in computing mac */
- crypto_cipher_encrypt_one(cipher, odata, odata);
+ sg_init_table(sg, 3);
+ sg_set_buf(&sg[0], odata, 16);
/* format associated data and compute into mac */
if (assoclen) {
- pctx->ilen = format_adata(idata, assoclen);
- get_data_to_compute(cipher, pctx, req->src, req->assoclen);
+ ilen = format_adata(idata, assoclen);
+ sg_set_buf(&sg[1], idata, ilen);
+ sg_chain(sg, 3, req->src);
} else {
- pctx->ilen = 0;
+ ilen = 0;
+ sg_chain(sg, 2, req->src);
}
- /* compute plaintext into mac */
- if (cryptlen)
- get_data_to_compute(cipher, pctx, plain, cryptlen);
+ ahash_request_set_tfm(ahreq, ctx->mac);
+ ahash_request_set_crypt(ahreq, sg, NULL, assoclen + ilen + 16);
+ err = crypto_ahash_init(ahreq);
+ if (err)
+ goto out;
+ err = crypto_ahash_update(ahreq);
+ if (err)
+ goto out;
+ /* we need to pad the MAC input to a round multiple of the block size */
+ ilen = 16 - (assoclen + ilen) % 16;
+ if (ilen < 16) {
+ memset(idata, 0, ilen);
+ sg_init_table(sg, 2);
+ sg_set_buf(&sg[0], idata, ilen);
+ sg_chain(sg, 2, plain);
+ plain = sg;
+ cryptlen += ilen;
+ }
+
+ ahash_request_set_crypt(ahreq, plain, pctx->odata, cryptlen);
+ err = crypto_ahash_finup(ahreq);
out:
return err;
}
@@ -453,21 +402,21 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
struct aead_instance *inst = aead_alg_instance(tfm);
struct ccm_instance_ctx *ictx = aead_instance_ctx(inst);
struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm);
- struct crypto_cipher *cipher;
+ struct crypto_ahash *mac;
struct crypto_skcipher *ctr;
unsigned long align;
int err;
- cipher = crypto_spawn_cipher(&ictx->cipher);
- if (IS_ERR(cipher))
- return PTR_ERR(cipher);
+ mac = crypto_spawn_ahash(&ictx->mac);
+ if (IS_ERR(mac))
+ return PTR_ERR(mac);
ctr = crypto_spawn_skcipher(&ictx->ctr);
err = PTR_ERR(ctr);
if (IS_ERR(ctr))
- goto err_free_cipher;
+ goto err_free_mac;
- ctx->cipher = cipher;
+ ctx->mac = mac;
ctx->ctr = ctr;
align = crypto_aead_alignmask(tfm);
@@ -479,8 +428,8 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
return 0;
-err_free_cipher:
- crypto_free_cipher(cipher);
+err_free_mac:
+ crypto_free_ahash(mac);
return err;
}
@@ -488,7 +437,7 @@ static void crypto_ccm_exit_tfm(struct crypto_aead *tfm)
{
struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm);
- crypto_free_cipher(ctx->cipher);
+ crypto_free_ahash(ctx->mac);
crypto_free_skcipher(ctx->ctr);
}
@@ -496,7 +445,7 @@ static void crypto_ccm_free(struct aead_instance *inst)
{
struct ccm_instance_ctx *ctx = aead_instance_ctx(inst);
- crypto_drop_spawn(&ctx->cipher);
+ crypto_drop_ahash(&ctx->mac);
crypto_drop_skcipher(&ctx->ctr);
kfree(inst);
}
@@ -505,12 +454,13 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
struct rtattr **tb,
const char *full_name,
const char *ctr_name,
- const char *cipher_name)
+ const char *mac_name)
{
struct crypto_attr_type *algt;
struct aead_instance *inst;
struct skcipher_alg *ctr;
- struct crypto_alg *cipher;
+ struct crypto_alg *mac_alg;
+ struct hash_alg_common *mac;
struct ccm_instance_ctx *ictx;
int err;
@@ -521,25 +471,26 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
return -EINVAL;
- cipher = crypto_alg_mod_lookup(cipher_name, CRYPTO_ALG_TYPE_CIPHER,
- CRYPTO_ALG_TYPE_MASK);
- if (IS_ERR(cipher))
- return PTR_ERR(cipher);
+ mac_alg = crypto_find_alg(mac_name, &crypto_ahash_type,
+ CRYPTO_ALG_TYPE_HASH,
+ CRYPTO_ALG_TYPE_AHASH_MASK |
+ CRYPTO_ALG_ASYNC);
+ if (IS_ERR(mac_alg))
+ return PTR_ERR(mac_alg);
+ mac = __crypto_hash_alg_common(mac_alg);
err = -EINVAL;
- if (cipher->cra_blocksize != 16)
- goto out_put_cipher;
+ if (mac->digestsize != 16)
+ goto out_put_mac;
inst = kzalloc(sizeof(*inst) + sizeof(*ictx), GFP_KERNEL);
err = -ENOMEM;
if (!inst)
- goto out_put_cipher;
+ goto out_put_mac;
ictx = aead_instance_ctx(inst);
-
- err = crypto_init_spawn(&ictx->cipher, cipher,
- aead_crypto_instance(inst),
- CRYPTO_ALG_TYPE_MASK);
+ err = crypto_init_ahash_spawn(&ictx->mac, mac,
+ aead_crypto_instance(inst));
if (err)
goto err_free_inst;
@@ -548,7 +499,7 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
crypto_requires_sync(algt->type,
algt->mask));
if (err)
- goto err_drop_cipher;
+ goto err_drop_mac;
ctr = crypto_spawn_skcipher_alg(&ictx->ctr);
@@ -564,16 +515,16 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
err = -ENAMETOOLONG;
if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
"ccm_base(%s,%s)", ctr->base.cra_driver_name,
- cipher->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
+ mac->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
goto err_drop_ctr;
memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
inst->alg.base.cra_flags = ctr->base.cra_flags & CRYPTO_ALG_ASYNC;
- inst->alg.base.cra_priority = (cipher->cra_priority +
+ inst->alg.base.cra_priority = (mac->base.cra_priority +
ctr->base.cra_priority) / 2;
inst->alg.base.cra_blocksize = 1;
- inst->alg.base.cra_alignmask = cipher->cra_alignmask |
+ inst->alg.base.cra_alignmask = mac->base.cra_alignmask |
ctr->base.cra_alignmask |
(__alignof__(u32) - 1);
inst->alg.ivsize = 16;
@@ -593,23 +544,24 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
if (err)
goto err_drop_ctr;
-out_put_cipher:
- crypto_mod_put(cipher);
+out_put_mac:
+ crypto_mod_put(mac_alg);
return err;
err_drop_ctr:
crypto_drop_skcipher(&ictx->ctr);
-err_drop_cipher:
- crypto_drop_spawn(&ictx->cipher);
+err_drop_mac:
+ crypto_drop_ahash(&ictx->mac);
err_free_inst:
kfree(inst);
- goto out_put_cipher;
+ goto out_put_mac;
}
static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb)
{
const char *cipher_name;
char ctr_name[CRYPTO_MAX_ALG_NAME];
+ char mac_name[CRYPTO_MAX_ALG_NAME];
char full_name[CRYPTO_MAX_ALG_NAME];
cipher_name = crypto_attr_alg_name(tb[1]);
@@ -620,12 +572,16 @@ static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb)
cipher_name) >= CRYPTO_MAX_ALG_NAME)
return -ENAMETOOLONG;
+ if (snprintf(mac_name, CRYPTO_MAX_ALG_NAME, "cbcmac(%s)",
+ cipher_name) >= CRYPTO_MAX_ALG_NAME)
+ return -ENAMETOOLONG;
+
if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm(%s)", cipher_name) >=
CRYPTO_MAX_ALG_NAME)
return -ENAMETOOLONG;
return crypto_ccm_create_common(tmpl, tb, full_name, ctr_name,
- cipher_name);
+ mac_name);
}
static struct crypto_template crypto_ccm_tmpl = {
@@ -899,14 +855,156 @@ static struct crypto_template crypto_rfc4309_tmpl = {
.module = THIS_MODULE,
};
+static int crypto_cbcmac_digest_setkey(struct crypto_shash *parent,
+ const u8 *inkey, unsigned int keylen)
+{
+ struct cbcmac_tfm_ctx *ctx = crypto_shash_ctx(parent);
+
+ return crypto_cipher_setkey(ctx->child, inkey, keylen);
+}
+
+static int crypto_cbcmac_digest_init(struct shash_desc *pdesc)
+{
+ struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
+ int bs = crypto_shash_digestsize(pdesc->tfm);
+
+ memset(ctx->dg, 0, bs);
+ ctx->len = 0;
+
+ return 0;
+}
+
+static int crypto_cbcmac_digest_update(struct shash_desc *pdesc, const u8 *p,
+ unsigned int len)
+{
+ struct crypto_shash *parent = pdesc->tfm;
+ struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent);
+ struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
+ struct crypto_cipher *tfm = tctx->child;
+ int bs = crypto_shash_digestsize(parent);
+
+ while (len--) {
+ ctx->dg[ctx->len++] ^= *p++;
+
+ if (ctx->len == bs) {
+ crypto_cipher_encrypt_one(tfm, ctx->dg, ctx->dg);
+ ctx->len = 0;
+ }
+ }
+
+ return 0;
+}
+
+static int crypto_cbcmac_digest_final(struct shash_desc *pdesc, u8 *out)
+{
+ struct crypto_shash *parent = pdesc->tfm;
+ struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent);
+ struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
+ struct crypto_cipher *tfm = tctx->child;
+ int bs = crypto_shash_digestsize(parent);
+
+ if (ctx->len)
+ crypto_cipher_encrypt_one(tfm, out, ctx->dg);
+ else
+ memcpy(out, ctx->dg, bs);
+
+ return 0;
+}
+
+static int cbcmac_init_tfm(struct crypto_tfm *tfm)
+{
+ struct crypto_cipher *cipher;
+ struct crypto_instance *inst = (void *)tfm->__crt_alg;
+ struct crypto_spawn *spawn = crypto_instance_ctx(inst);
+ struct cbcmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ cipher = crypto_spawn_cipher(spawn);
+ if (IS_ERR(cipher))
+ return PTR_ERR(cipher);
+
+ ctx->child = cipher;
+
+ return 0;
+};
+
+static void cbcmac_exit_tfm(struct crypto_tfm *tfm)
+{
+ struct cbcmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+ crypto_free_cipher(ctx->child);
+}
+
+static int cbcmac_create(struct crypto_template *tmpl, struct rtattr **tb)
+{
+ struct shash_instance *inst;
+ struct crypto_alg *alg;
+ int err;
+
+ err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
+ if (err)
+ return err;
+
+ alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
+ CRYPTO_ALG_TYPE_MASK);
+ if (IS_ERR(alg))
+ return PTR_ERR(alg);
+
+ inst = shash_alloc_instance("cbcmac", alg);
+ err = PTR_ERR(inst);
+ if (IS_ERR(inst))
+ goto out_put_alg;
+
+ err = crypto_init_spawn(shash_instance_ctx(inst), alg,
+ shash_crypto_instance(inst),
+ CRYPTO_ALG_TYPE_MASK);
+ if (err)
+ goto out_free_inst;
+
+ inst->alg.base.cra_priority = alg->cra_priority;
+ inst->alg.base.cra_blocksize = 1;
+
+ inst->alg.digestsize = alg->cra_blocksize;
+ inst->alg.descsize = sizeof(struct cbcmac_desc_ctx) +
+ alg->cra_blocksize;
+
+ inst->alg.base.cra_ctxsize = sizeof(struct cbcmac_tfm_ctx);
+ inst->alg.base.cra_init = cbcmac_init_tfm;
+ inst->alg.base.cra_exit = cbcmac_exit_tfm;
+
+ inst->alg.init = crypto_cbcmac_digest_init;
+ inst->alg.update = crypto_cbcmac_digest_update;
+ inst->alg.final = crypto_cbcmac_digest_final;
+ inst->alg.setkey = crypto_cbcmac_digest_setkey;
+
+ err = shash_register_instance(tmpl, inst);
+
+out_free_inst:
+ if (err)
+ shash_free_instance(shash_crypto_instance(inst));
+
+out_put_alg:
+ crypto_mod_put(alg);
+ return err;
+}
+
+static struct crypto_template crypto_cbcmac_tmpl = {
+ .name = "cbcmac",
+ .create = cbcmac_create,
+ .free = shash_free_instance,
+ .module = THIS_MODULE,
+};
+
static int __init crypto_ccm_module_init(void)
{
int err;
- err = crypto_register_template(&crypto_ccm_base_tmpl);
+ err = crypto_register_template(&crypto_cbcmac_tmpl);
if (err)
goto out;
+ err = crypto_register_template(&crypto_ccm_base_tmpl);
+ if (err)
+ goto out_undo_cbcmac;
+
err = crypto_register_template(&crypto_ccm_tmpl);
if (err)
goto out_undo_base;
@@ -922,6 +1020,8 @@ static int __init crypto_ccm_module_init(void)
crypto_unregister_template(&crypto_ccm_tmpl);
out_undo_base:
crypto_unregister_template(&crypto_ccm_base_tmpl);
+out_undo_cbcmac:
+ crypto_register_template(&crypto_cbcmac_tmpl);
goto out;
}
@@ -930,6 +1030,7 @@ static void __exit crypto_ccm_module_exit(void)
crypto_unregister_template(&crypto_rfc4309_tmpl);
crypto_unregister_template(&crypto_ccm_tmpl);
crypto_unregister_template(&crypto_ccm_base_tmpl);
+ crypto_unregister_template(&crypto_cbcmac_tmpl);
}
module_init(crypto_ccm_module_init);
--
2.7.4
^ permalink raw reply related
* [PATCH 3/4] crypto: arm64/aes - add NEON and Crypto Extension CBC-MAC driver
From: Ard Biesheuvel @ 2017-01-26 17:17 UTC (permalink / raw)
To: linux-crypto, linux-arm-kernel; +Cc: herbert, Ard Biesheuvel
In-Reply-To: <1485451063-11822-1-git-send-email-ard.biesheuvel@linaro.org>
On ARMv8 implementations that do not support the Crypto Extensions,
such as the Raspberry Pi 3, the CCM driver falls back to the generic
table based AES implementation to perform the MAC part of the
algorithm, which is slow and not time invariant. So add a CBCMAC
implementation to the shared glue code between NEON AES and Crypto
Extensions AES, so that it can be used instead now that the CCM
driver has been updated to look for CBCMAC implementations other
than the one it supplies itself.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/aes-glue.c | 102 ++++++++++++++++++++
arch/arm64/crypto/aes-modes.S | 19 ++++
2 files changed, 121 insertions(+)
diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index 055bc3f61138..1f29570b83e9 100644
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -11,6 +11,7 @@
#include <asm/neon.h>
#include <asm/hwcap.h>
#include <crypto/aes.h>
+#include <crypto/internal/hash.h>
#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
#include <linux/module.h>
@@ -31,6 +32,7 @@
#define aes_ctr_encrypt ce_aes_ctr_encrypt
#define aes_xts_encrypt ce_aes_xts_encrypt
#define aes_xts_decrypt ce_aes_xts_decrypt
+#define aes_cbcmac_update ce_aes_cbcmac_update
MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
#else
#define MODE "neon"
@@ -44,11 +46,13 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
#define aes_ctr_encrypt neon_aes_ctr_encrypt
#define aes_xts_encrypt neon_aes_xts_encrypt
#define aes_xts_decrypt neon_aes_xts_decrypt
+#define aes_cbcmac_update neon_aes_cbcmac_update
MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");
MODULE_ALIAS_CRYPTO("ecb(aes)");
MODULE_ALIAS_CRYPTO("cbc(aes)");
MODULE_ALIAS_CRYPTO("ctr(aes)");
MODULE_ALIAS_CRYPTO("xts(aes)");
+MODULE_ALIAS_CRYPTO("cbcmac(aes)");
#endif
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
@@ -75,11 +79,19 @@ asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[],
int rounds, int blocks, u8 const rk2[], u8 iv[],
int first);
+asmlinkage void aes_cbcmac_update(u8 const in[], u32 const rk[], int rounds,
+ int blocks, u8 dg[]);
+
struct crypto_aes_xts_ctx {
struct crypto_aes_ctx key1;
struct crypto_aes_ctx __aligned(8) key2;
};
+struct cbcmac_desc_ctx {
+ unsigned int len;
+ u8 dg[];
+};
+
static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
unsigned int key_len)
{
@@ -357,6 +369,89 @@ static struct skcipher_alg aes_algs[] = { {
.decrypt = xts_decrypt,
} };
+static int cbcmac_setkey(struct crypto_shash *tfm,
+ const u8 *in_key, unsigned int key_len)
+{
+ struct crypto_aes_ctx *ctx = crypto_shash_ctx(tfm);
+ int err;
+
+ err = aes_expandkey(ctx, in_key, key_len);
+ if (err)
+ crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+
+ return err;
+}
+
+static int cbcmac_init(struct shash_desc *desc)
+{
+ struct cbcmac_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ memset(ctx->dg, 0, AES_BLOCK_SIZE);
+ ctx->len = 0;
+
+ return 0;
+}
+
+static int cbcmac_update(struct shash_desc *desc, const u8 *p,
+ unsigned int len)
+{
+ struct crypto_aes_ctx *tctx = crypto_shash_ctx(desc->tfm);
+ struct cbcmac_desc_ctx *ctx = shash_desc_ctx(desc);
+ int rounds = 6 + tctx->key_length / 4;
+
+ while (len--) {
+ ctx->dg[ctx->len++] ^= *p++;
+
+ if (ctx->len == AES_BLOCK_SIZE) {
+ int blocks = len / AES_BLOCK_SIZE;
+
+ kernel_neon_begin();
+ aes_cbcmac_update(p, tctx->key_enc, rounds, blocks,
+ ctx->dg);
+ kernel_neon_end();
+
+ ctx->len = 0;
+ len %= AES_BLOCK_SIZE;
+ p += blocks * AES_BLOCK_SIZE;
+ }
+ }
+
+ return 0;
+}
+
+static int cbcmac_final(struct shash_desc *desc, u8 *out)
+{
+ struct crypto_aes_ctx *tctx = crypto_shash_ctx(desc->tfm);
+ struct cbcmac_desc_ctx *ctx = shash_desc_ctx(desc);
+ int rounds = 6 + tctx->key_length / 4;
+
+ if (ctx->len) {
+ kernel_neon_begin();
+ aes_cbcmac_update(NULL, tctx->key_enc, rounds, 0, ctx->dg);
+ kernel_neon_end();
+ }
+ memcpy(out, ctx->dg, AES_BLOCK_SIZE);
+
+ return 0;
+}
+
+static struct shash_alg cbcmac_alg = {
+ .base.cra_name = "cbcmac(aes)",
+ .base.cra_driver_name = "cbcmac-aes-" MODE,
+ .base.cra_priority = PRIO,
+ .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
+ .base.cra_module = THIS_MODULE,
+
+ .digestsize = AES_BLOCK_SIZE,
+ .init = cbcmac_init,
+ .update = cbcmac_update,
+ .final = cbcmac_final,
+ .setkey = cbcmac_setkey,
+ .descsize = sizeof(struct cbcmac_desc_ctx),
+};
+
static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
static void aes_exit(void)
@@ -367,6 +462,7 @@ static void aes_exit(void)
if (aes_simd_algs[i])
simd_skcipher_free(aes_simd_algs[i]);
+ crypto_unregister_shash(&cbcmac_alg);
crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
}
@@ -383,6 +479,10 @@ static int __init aes_init(void)
if (err)
return err;
+ err = crypto_register_shash(&cbcmac_alg);
+ if (err)
+ goto unregister_ciphers;
+
for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
continue;
@@ -402,6 +502,8 @@ static int __init aes_init(void)
unregister_simds:
aes_exit();
+unregister_ciphers:
+ crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
return err;
}
diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S
index 92b982a8b112..aa96c9691af9 100644
--- a/arch/arm64/crypto/aes-modes.S
+++ b/arch/arm64/crypto/aes-modes.S
@@ -525,3 +525,22 @@ AES_ENTRY(aes_xts_decrypt)
FRAME_POP
ret
AES_ENDPROC(aes_xts_decrypt)
+
+ /*
+ * aes_cbcmac_update(u8 const in[], u32 const rk[], int rounds,
+ * int blocks, u8 dg[])
+ */
+AES_ENTRY(aes_cbcmac_update)
+ ld1 {v0.16b}, [x4] /* get iv */
+ enc_prepare w2, x1, x5
+
+.Lcbcmacloop:
+ ld1 {v1.16b}, [x0], #16 /* get next pt block */
+ eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */
+ encrypt_block v0, w2, x1, x5, w6
+ subs w3, w3, #1
+ bne .Lcbcmacloop
+
+ st1 {v0.16b}, [x4] /* return iv */
+ ret
+AES_ENDPROC(aes_cbcmac_update)
--
2.7.4
^ permalink raw reply related
* [RFC PATCH 4/4] crypto: aes - add generic time invariant AES for CTR/CCM/GCM
From: Ard Biesheuvel @ 2017-01-26 17:17 UTC (permalink / raw)
To: linux-crypto, linux-arm-kernel; +Cc: herbert, Ard Biesheuvel
In-Reply-To: <1485451063-11822-1-git-send-email-ard.biesheuvel@linaro.org>
Lookup table based AES is sensitive to timing attacks, which is
due to the fact that such table lookups are data dependent, and
the fact that 8 KB worth of tables covers a significant number of
cachelines on any architecture.
For network facing algorithms such as CTR, CCM or GCM, this presents
a security risk, which is why arch specific AES ports are typically
time invariant, either through the use of special instructions, or
by using SIMD algorithms that don't rely on table lookups.
For generic code, this is difficult to achieve without losing too
much performance, but we can improve the situation significantly by
switching to an implementation that only needs 256 bytes of table
data (the actual S-box itself), which can be prefetched at the start
of each block to eliminate data dependent latencies.
Note that this only implements AES encryption, which is all we need
for CTR and CBC-MAC. AES decryption can easily be implemented in a
similar way, but is significantly more costly.
This code runs at ~25 cycles per byte on ARM Cortex-A57 (while the
ordinary generic AES driver manages 18 cycles per byte on this
hardware).
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
crypto/Kconfig | 14 +
crypto/Makefile | 1 +
crypto/aes_ti.c | 314 ++++++++++++++++++++
3 files changed, 329 insertions(+)
diff --git a/crypto/Kconfig b/crypto/Kconfig
index e8269d1b0282..ce1f6be9e48f 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -896,6 +896,20 @@ config CRYPTO_AES
See <http://csrc.nist.gov/CryptoToolkit/aes/> for more information.
+config CRYPTO_AES_TI
+ tristate "Generic time invariant AES in CTR and CBC-MAC modes"
+ select CRYPTO_BLKCIPHER
+ select CRYPTO_HASH
+ select CRYPTO_AES
+ help
+ This is a time invariant generic implementation of AES in CTR and
+ CBC-MAC modes, intended for use by the generic CCM and GCM drivers,
+ and other CTR based modes. Instead of using 8 lookup tables of 1 KB
+ each, both for encryption and decryption, this implementation only
+ uses a single S-box of 256 bytes, and attempts to eliminate data
+ dependent latencies by prefetching the entire table into the cache
+ at the start of each block.
+
config CRYPTO_AES_586
tristate "AES cipher algorithms (i586)"
depends on (X86 || UML_X86) && !64BIT
diff --git a/crypto/Makefile b/crypto/Makefile
index b8f0e3eb0791..bcd834536163 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -99,6 +99,7 @@ obj-$(CONFIG_CRYPTO_TWOFISH) += twofish_generic.o
obj-$(CONFIG_CRYPTO_TWOFISH_COMMON) += twofish_common.o
obj-$(CONFIG_CRYPTO_SERPENT) += serpent_generic.o
obj-$(CONFIG_CRYPTO_AES) += aes_generic.o
+obj-$(CONFIG_CRYPTO_AES_TI) += aes_ti.o
obj-$(CONFIG_CRYPTO_CAMELLIA) += camellia_generic.o
obj-$(CONFIG_CRYPTO_CAST_COMMON) += cast_common.o
obj-$(CONFIG_CRYPTO_CAST5) += cast5_generic.o
diff --git a/crypto/aes_ti.c b/crypto/aes_ti.c
new file mode 100644
index 000000000000..5ad80e063681
--- /dev/null
+++ b/crypto/aes_ti.c
@@ -0,0 +1,314 @@
+/*
+ * Scalar (mostly) time invariant AES core transform for CTR/CCM/GCM
+ *
+ * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <crypto/aes.h>
+#include <crypto/internal/hash.h>
+#include <crypto/internal/skcipher.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+#include <asm/unaligned.h>
+
+struct aes_ti_ctx {
+ u32 rk[AES_MAX_KEYLENGTH_U32];
+ int rounds;
+};
+
+struct cbcmac_desc_ctx {
+ unsigned int len;
+ u8 dg[];
+};
+
+__weak const u8 __cacheline_aligned __aesti_sbox[] = {
+ 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5,
+ 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
+ 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0,
+ 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
+ 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc,
+ 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
+ 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a,
+ 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
+ 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0,
+ 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
+ 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b,
+ 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
+ 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85,
+ 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
+ 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5,
+ 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
+ 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17,
+ 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
+ 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88,
+ 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
+ 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c,
+ 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
+ 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9,
+ 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
+ 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6,
+ 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
+ 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e,
+ 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
+ 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94,
+ 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
+ 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68,
+ 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16,
+};
+
+static int aesti_set_key(struct aes_ti_ctx *ctx, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct crypto_aes_ctx rk;
+ int err;
+
+ err = crypto_aes_expand_key(&rk, in_key, key_len);
+ if (err)
+ return err;
+
+ memcpy(ctx->rk, rk.key_enc, sizeof(ctx->rk));
+ ctx->rounds = 6 + key_len / 4;
+
+ /*
+ * In order to force the compiler to emit data independent Sbox lookups
+ * at the start of each block, xor the first round key with values at
+ * fixed indexes in the Sbox.
+ */
+ ctx->rk[0] ^= __aesti_sbox[ 0] ^ __aesti_sbox[128];
+ ctx->rk[1] ^= __aesti_sbox[32] ^ __aesti_sbox[160];
+ ctx->rk[2] ^= __aesti_sbox[64] ^ __aesti_sbox[192];
+ ctx->rk[3] ^= __aesti_sbox[96] ^ __aesti_sbox[224];
+
+ return 0;
+}
+
+static u32 mul_by_x(u32 w)
+{
+ /* multiply by polynomial 'x' (0b10) in GF(2^8) */
+ return ((w & 0x80808080) >> 7) * 0x1b ^ ((w & 0x7f7f7f7f) << 1);
+}
+
+static u32 mix_columns(u32 x)
+{
+ u32 y = mul_by_x(x) ^ ror32(x, 16);
+
+ return y ^ ror32(x ^ y, 8);
+}
+
+static __always_inline u32 subshift(u32 in[], int pos)
+{
+ return (__aesti_sbox[in[pos] & 0xff]) ^
+ (__aesti_sbox[(in[(pos + 1) % 4] >> 8) & 0xff] << 8) ^
+ (__aesti_sbox[(in[(pos + 2) % 4] >> 16) & 0xff] << 16) ^
+ (__aesti_sbox[(in[(pos + 3) % 4] >> 24) & 0xff] << 24);
+}
+
+static void aesti_encrypt(struct aes_ti_ctx *ctx, u8 *out, const u8 *in)
+{
+ u32 st0[4], st1[4];
+ u32 *rkp = ctx->rk + 4;
+ int round;
+
+ st0[0] = get_unaligned_le32(in);
+ st0[1] = get_unaligned_le32(in + 4);
+ st0[2] = get_unaligned_le32(in + 8);
+ st0[3] = get_unaligned_le32(in + 12);
+
+ st0[0] ^= __aesti_sbox[ 0] ^ __aesti_sbox[128] ^ ctx->rk[0];
+ st0[1] ^= __aesti_sbox[32] ^ __aesti_sbox[160] ^ ctx->rk[1];
+ st0[2] ^= __aesti_sbox[64] ^ __aesti_sbox[192] ^ ctx->rk[2];
+ st0[3] ^= __aesti_sbox[96] ^ __aesti_sbox[224] ^ ctx->rk[3];
+
+ for (round = 0;; round += 2) {
+ st1[0] = mix_columns(subshift(st0, 0)) ^ *rkp++;
+ st1[1] = mix_columns(subshift(st0, 1)) ^ *rkp++;
+ st1[2] = mix_columns(subshift(st0, 2)) ^ *rkp++;
+ st1[3] = mix_columns(subshift(st0, 3)) ^ *rkp++;
+
+ if (round == ctx->rounds - 2)
+ break;
+
+ st0[0] = mix_columns(subshift(st1, 0)) ^ *rkp++;
+ st0[1] = mix_columns(subshift(st1, 1)) ^ *rkp++;
+ st0[2] = mix_columns(subshift(st1, 2)) ^ *rkp++;
+ st0[3] = mix_columns(subshift(st1, 3)) ^ *rkp++;
+ }
+
+ put_unaligned_le32(subshift(st1, 0) ^ rkp[0], out);
+ put_unaligned_le32(subshift(st1, 1) ^ rkp[1], out + 4);
+ put_unaligned_le32(subshift(st1, 2) ^ rkp[2], out + 8);
+ put_unaligned_le32(subshift(st1, 3) ^ rkp[3], out + 12);
+}
+
+static int aesti_ctr_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct aes_ti_ctx *ctx = crypto_skcipher_ctx(tfm);
+ int err;
+
+ err = aesti_set_key(ctx, in_key, key_len);
+ if (err)
+ crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return err;
+}
+
+static int aesti_ctr_encrypt(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct aes_ti_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ u8 buf[AES_BLOCK_SIZE];
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ while (walk.nbytes > 0) {
+ u8 *dst = walk.dst.virt.addr;
+ u8 *src = walk.src.virt.addr;
+ int nbytes = walk.nbytes;
+ int tail = 0;
+
+ if (nbytes < walk.total) {
+ nbytes = round_down(nbytes, AES_BLOCK_SIZE);
+ tail = walk.nbytes % AES_BLOCK_SIZE;
+ }
+
+ do {
+ int bsize = min(nbytes, AES_BLOCK_SIZE);
+
+ aesti_encrypt(ctx, buf, walk.iv);
+ if (dst != src)
+ memcpy(dst, src, bsize);
+ crypto_xor(dst, buf, bsize);
+ crypto_inc(walk.iv, AES_BLOCK_SIZE);
+
+ dst += AES_BLOCK_SIZE;
+ src += AES_BLOCK_SIZE;
+ nbytes -= AES_BLOCK_SIZE;
+ } while (nbytes > 0);
+
+ err = skcipher_walk_done(&walk, tail);
+ }
+ return err;
+}
+
+static struct skcipher_alg ctr_alg = {
+ .base.cra_name = "ctr(aes)",
+ .base.cra_driver_name = "ctr-aes-ti",
+ .base.cra_priority = 100 + 1,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct aes_ti_ctx),
+ .base.cra_module = THIS_MODULE,
+
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .chunksize = AES_BLOCK_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aesti_ctr_set_key,
+ .encrypt = aesti_ctr_encrypt,
+ .decrypt = aesti_ctr_encrypt,
+};
+
+static int aesti_cbcmac_setkey(struct crypto_shash *tfm,
+ const u8 *in_key, unsigned int key_len)
+{
+ struct aes_ti_ctx *ctx = crypto_shash_ctx(tfm);
+ int err;
+
+ err = aesti_set_key(ctx, in_key, key_len);
+ if (err)
+ crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+
+ return err;
+}
+
+static int aesti_cbcmac_init(struct shash_desc *desc)
+{
+ struct cbcmac_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ memset(ctx->dg, 0, AES_BLOCK_SIZE);
+ ctx->len = 0;
+
+ return 0;
+}
+
+static int aesti_cbcmac_update(struct shash_desc *desc, const u8 *p,
+ unsigned int len)
+{
+ struct aes_ti_ctx *tctx = crypto_shash_ctx(desc->tfm);
+ struct cbcmac_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ while (len--) {
+ ctx->dg[ctx->len++] ^= *p++;
+
+ if (ctx->len == AES_BLOCK_SIZE) {
+ aesti_encrypt(tctx, ctx->dg, ctx->dg);
+ ctx->len = 0;
+ }
+ }
+
+ return 0;
+}
+
+static int aesti_cbcmac_final(struct shash_desc *desc, u8 *out)
+{
+ struct aes_ti_ctx *tctx = crypto_shash_ctx(desc->tfm);
+ struct cbcmac_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ if (ctx->len)
+ aesti_encrypt(tctx, out, ctx->dg);
+ else
+ memcpy(out, ctx->dg, AES_BLOCK_SIZE);
+
+ return 0;
+}
+
+static struct shash_alg cbcmac_alg = {
+ .base.cra_name = "cbcmac(aes)",
+ .base.cra_driver_name = "cbcmac-aes-ti",
+ .base.cra_priority = 100 + 1,
+ .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct aes_ti_ctx),
+ .base.cra_module = THIS_MODULE,
+
+ .digestsize = AES_BLOCK_SIZE,
+ .init = aesti_cbcmac_init,
+ .update = aesti_cbcmac_update,
+ .final = aesti_cbcmac_final,
+ .setkey = aesti_cbcmac_setkey,
+ .descsize = sizeof(struct cbcmac_desc_ctx),
+};
+
+static int __init aes_init(void)
+{
+ int err;
+
+ err = crypto_register_skcipher(&ctr_alg);
+ if (err)
+ return err;
+
+ err = crypto_register_shash(&cbcmac_alg);
+ if (err)
+ crypto_unregister_skcipher(&ctr_alg);
+ return err;
+}
+
+static void __exit aes_fini(void)
+{
+ crypto_unregister_shash(&cbcmac_alg);
+ crypto_unregister_skcipher(&ctr_alg);
+}
+
+module_init(aes_init);
+module_exit(aes_fini);
+
+MODULE_DESCRIPTION("Generic time invariant AES transform in CTR and CBC-MAC modes");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("cbcmac(aes)");
+MODULE_ALIAS_CRYPTO("ctr(aes)");
--
2.7.4
^ permalink raw reply related
* Re: [RFC PATCH 4/4] crypto: aes - add generic time invariant AES for CTR/CCM/GCM
From: Krzysztof Kwiatkowski @ 2017-01-26 18:35 UTC (permalink / raw)
To: Ard Biesheuvel, linux-crypto, linux-arm-kernel; +Cc: herbert
In-Reply-To: <1485451063-11822-5-git-send-email-ard.biesheuvel@linaro.org>
Ard,
This is really interesting implementation. Is there a way to test if
execution of this code is really constant time. Have you done any tests
like that? Adam Langley has proposed using modified version of valgrind
(ctgrind) for that, but I wonder if you maybe thought about any
alternative method?
Kind regards,
Kris
On 26/01/17 17:17, Ard Biesheuvel wrote:
> Lookup table based AES is sensitive to timing attacks, which is
> due to the fact that such table lookups are data dependent, and
> the fact that 8 KB worth of tables covers a significant number of
> cachelines on any architecture.
>
> For network facing algorithms such as CTR, CCM or GCM, this presents
> a security risk, which is why arch specific AES ports are typically
> time invariant, either through the use of special instructions, or
> by using SIMD algorithms that don't rely on table lookups.
>
> For generic code, this is difficult to achieve without losing too
> much performance, but we can improve the situation significantly by
> switching to an implementation that only needs 256 bytes of table
> data (the actual S-box itself), which can be prefetched at the start
> of each block to eliminate data dependent latencies.
>
> Note that this only implements AES encryption, which is all we need
> for CTR and CBC-MAC. AES decryption can easily be implemented in a
> similar way, but is significantly more costly.
>
> This code runs at ~25 cycles per byte on ARM Cortex-A57 (while the
> ordinary generic AES driver manages 18 cycles per byte on this
> hardware).
>
> Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
> ---
> crypto/Kconfig | 14 +
> crypto/Makefile | 1 +
> crypto/aes_ti.c | 314 ++++++++++++++++++++
> 3 files changed, 329 insertions(+)
>
> diff --git a/crypto/Kconfig b/crypto/Kconfig
> index e8269d1b0282..ce1f6be9e48f 100644
> --- a/crypto/Kconfig
> +++ b/crypto/Kconfig
> @@ -896,6 +896,20 @@ config CRYPTO_AES
>
> See <http://csrc.nist.gov/CryptoToolkit/aes/> for more information.
>
> +config CRYPTO_AES_TI
> + tristate "Generic time invariant AES in CTR and CBC-MAC modes"
> + select CRYPTO_BLKCIPHER
> + select CRYPTO_HASH
> + select CRYPTO_AES
> + help
> + This is a time invariant generic implementation of AES in CTR and
> + CBC-MAC modes, intended for use by the generic CCM and GCM drivers,
> + and other CTR based modes. Instead of using 8 lookup tables of 1 KB
> + each, both for encryption and decryption, this implementation only
> + uses a single S-box of 256 bytes, and attempts to eliminate data
> + dependent latencies by prefetching the entire table into the cache
> + at the start of each block.
> +
> config CRYPTO_AES_586
> tristate "AES cipher algorithms (i586)"
> depends on (X86 || UML_X86) && !64BIT
> diff --git a/crypto/Makefile b/crypto/Makefile
> index b8f0e3eb0791..bcd834536163 100644
> --- a/crypto/Makefile
> +++ b/crypto/Makefile
> @@ -99,6 +99,7 @@ obj-$(CONFIG_CRYPTO_TWOFISH) += twofish_generic.o
> obj-$(CONFIG_CRYPTO_TWOFISH_COMMON) += twofish_common.o
> obj-$(CONFIG_CRYPTO_SERPENT) += serpent_generic.o
> obj-$(CONFIG_CRYPTO_AES) += aes_generic.o
> +obj-$(CONFIG_CRYPTO_AES_TI) += aes_ti.o
> obj-$(CONFIG_CRYPTO_CAMELLIA) += camellia_generic.o
> obj-$(CONFIG_CRYPTO_CAST_COMMON) += cast_common.o
> obj-$(CONFIG_CRYPTO_CAST5) += cast5_generic.o
> diff --git a/crypto/aes_ti.c b/crypto/aes_ti.c
> new file mode 100644
> index 000000000000..5ad80e063681
> --- /dev/null
> +++ b/crypto/aes_ti.c
> @@ -0,0 +1,314 @@
> +/*
> + * Scalar (mostly) time invariant AES core transform for CTR/CCM/GCM
> + *
> + * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + */
> +
> +#include <crypto/aes.h>
> +#include <crypto/internal/hash.h>
> +#include <crypto/internal/skcipher.h>
> +#include <linux/crypto.h>
> +#include <linux/module.h>
> +#include <asm/unaligned.h>
> +
> +struct aes_ti_ctx {
> + u32 rk[AES_MAX_KEYLENGTH_U32];
> + int rounds;
> +};
> +
> +struct cbcmac_desc_ctx {
> + unsigned int len;
> + u8 dg[];
> +};
> +
> +__weak const u8 __cacheline_aligned __aesti_sbox[] = {
> + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5,
> + 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
> + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0,
> + 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
> + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc,
> + 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
> + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a,
> + 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
> + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0,
> + 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
> + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b,
> + 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
> + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85,
> + 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
> + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5,
> + 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
> + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17,
> + 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
> + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88,
> + 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
> + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c,
> + 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
> + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9,
> + 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
> + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6,
> + 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
> + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e,
> + 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
> + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94,
> + 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
> + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68,
> + 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16,
> +};
> +
> +static int aesti_set_key(struct aes_ti_ctx *ctx, const u8 *in_key,
> + unsigned int key_len)
> +{
> + struct crypto_aes_ctx rk;
> + int err;
> +
> + err = crypto_aes_expand_key(&rk, in_key, key_len);
> + if (err)
> + return err;
> +
> + memcpy(ctx->rk, rk.key_enc, sizeof(ctx->rk));
> + ctx->rounds = 6 + key_len / 4;
> +
> + /*
> + * In order to force the compiler to emit data independent Sbox lookups
> + * at the start of each block, xor the first round key with values at
> + * fixed indexes in the Sbox.
> + */
> + ctx->rk[0] ^= __aesti_sbox[ 0] ^ __aesti_sbox[128];
> + ctx->rk[1] ^= __aesti_sbox[32] ^ __aesti_sbox[160];
> + ctx->rk[2] ^= __aesti_sbox[64] ^ __aesti_sbox[192];
> + ctx->rk[3] ^= __aesti_sbox[96] ^ __aesti_sbox[224];
> +
> + return 0;
> +}
> +
> +static u32 mul_by_x(u32 w)
> +{
> + /* multiply by polynomial 'x' (0b10) in GF(2^8) */
> + return ((w & 0x80808080) >> 7) * 0x1b ^ ((w & 0x7f7f7f7f) << 1);
> +}
> +
> +static u32 mix_columns(u32 x)
> +{
> + u32 y = mul_by_x(x) ^ ror32(x, 16);
> +
> + return y ^ ror32(x ^ y, 8);
> +}
> +
> +static __always_inline u32 subshift(u32 in[], int pos)
> +{
> + return (__aesti_sbox[in[pos] & 0xff]) ^
> + (__aesti_sbox[(in[(pos + 1) % 4] >> 8) & 0xff] << 8) ^
> + (__aesti_sbox[(in[(pos + 2) % 4] >> 16) & 0xff] << 16) ^
> + (__aesti_sbox[(in[(pos + 3) % 4] >> 24) & 0xff] << 24);
> +}
> +
> +static void aesti_encrypt(struct aes_ti_ctx *ctx, u8 *out, const u8 *in)
> +{
> + u32 st0[4], st1[4];
> + u32 *rkp = ctx->rk + 4;
> + int round;
> +
> + st0[0] = get_unaligned_le32(in);
> + st0[1] = get_unaligned_le32(in + 4);
> + st0[2] = get_unaligned_le32(in + 8);
> + st0[3] = get_unaligned_le32(in + 12);
> +
> + st0[0] ^= __aesti_sbox[ 0] ^ __aesti_sbox[128] ^ ctx->rk[0];
> + st0[1] ^= __aesti_sbox[32] ^ __aesti_sbox[160] ^ ctx->rk[1];
> + st0[2] ^= __aesti_sbox[64] ^ __aesti_sbox[192] ^ ctx->rk[2];
> + st0[3] ^= __aesti_sbox[96] ^ __aesti_sbox[224] ^ ctx->rk[3];
> +
> + for (round = 0;; round += 2) {
> + st1[0] = mix_columns(subshift(st0, 0)) ^ *rkp++;
> + st1[1] = mix_columns(subshift(st0, 1)) ^ *rkp++;
> + st1[2] = mix_columns(subshift(st0, 2)) ^ *rkp++;
> + st1[3] = mix_columns(subshift(st0, 3)) ^ *rkp++;
> +
> + if (round == ctx->rounds - 2)
> + break;
> +
> + st0[0] = mix_columns(subshift(st1, 0)) ^ *rkp++;
> + st0[1] = mix_columns(subshift(st1, 1)) ^ *rkp++;
> + st0[2] = mix_columns(subshift(st1, 2)) ^ *rkp++;
> + st0[3] = mix_columns(subshift(st1, 3)) ^ *rkp++;
> + }
> +
> + put_unaligned_le32(subshift(st1, 0) ^ rkp[0], out);
> + put_unaligned_le32(subshift(st1, 1) ^ rkp[1], out + 4);
> + put_unaligned_le32(subshift(st1, 2) ^ rkp[2], out + 8);
> + put_unaligned_le32(subshift(st1, 3) ^ rkp[3], out + 12);
> +}
> +
> +static int aesti_ctr_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
> + unsigned int key_len)
> +{
> + struct aes_ti_ctx *ctx = crypto_skcipher_ctx(tfm);
> + int err;
> +
> + err = aesti_set_key(ctx, in_key, key_len);
> + if (err)
> + crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
> + return err;
> +}
> +
> +static int aesti_ctr_encrypt(struct skcipher_request *req)
> +{
> + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
> + struct aes_ti_ctx *ctx = crypto_skcipher_ctx(tfm);
> + struct skcipher_walk walk;
> + u8 buf[AES_BLOCK_SIZE];
> + int err;
> +
> + err = skcipher_walk_virt(&walk, req, true);
> +
> + while (walk.nbytes > 0) {
> + u8 *dst = walk.dst.virt.addr;
> + u8 *src = walk.src.virt.addr;
> + int nbytes = walk.nbytes;
> + int tail = 0;
> +
> + if (nbytes < walk.total) {
> + nbytes = round_down(nbytes, AES_BLOCK_SIZE);
> + tail = walk.nbytes % AES_BLOCK_SIZE;
> + }
> +
> + do {
> + int bsize = min(nbytes, AES_BLOCK_SIZE);
> +
> + aesti_encrypt(ctx, buf, walk.iv);
> + if (dst != src)
> + memcpy(dst, src, bsize);
> + crypto_xor(dst, buf, bsize);
> + crypto_inc(walk.iv, AES_BLOCK_SIZE);
> +
> + dst += AES_BLOCK_SIZE;
> + src += AES_BLOCK_SIZE;
> + nbytes -= AES_BLOCK_SIZE;
> + } while (nbytes > 0);
> +
> + err = skcipher_walk_done(&walk, tail);
> + }
> + return err;
> +}
> +
> +static struct skcipher_alg ctr_alg = {
> + .base.cra_name = "ctr(aes)",
> + .base.cra_driver_name = "ctr-aes-ti",
> + .base.cra_priority = 100 + 1,
> + .base.cra_blocksize = 1,
> + .base.cra_ctxsize = sizeof(struct aes_ti_ctx),
> + .base.cra_module = THIS_MODULE,
> +
> + .min_keysize = AES_MIN_KEY_SIZE,
> + .max_keysize = AES_MAX_KEY_SIZE,
> + .chunksize = AES_BLOCK_SIZE,
> + .ivsize = AES_BLOCK_SIZE,
> + .setkey = aesti_ctr_set_key,
> + .encrypt = aesti_ctr_encrypt,
> + .decrypt = aesti_ctr_encrypt,
> +};
> +
> +static int aesti_cbcmac_setkey(struct crypto_shash *tfm,
> + const u8 *in_key, unsigned int key_len)
> +{
> + struct aes_ti_ctx *ctx = crypto_shash_ctx(tfm);
> + int err;
> +
> + err = aesti_set_key(ctx, in_key, key_len);
> + if (err)
> + crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
> +
> + return err;
> +}
> +
> +static int aesti_cbcmac_init(struct shash_desc *desc)
> +{
> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(desc);
> +
> + memset(ctx->dg, 0, AES_BLOCK_SIZE);
> + ctx->len = 0;
> +
> + return 0;
> +}
> +
> +static int aesti_cbcmac_update(struct shash_desc *desc, const u8 *p,
> + unsigned int len)
> +{
> + struct aes_ti_ctx *tctx = crypto_shash_ctx(desc->tfm);
> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(desc);
> +
> + while (len--) {
> + ctx->dg[ctx->len++] ^= *p++;
> +
> + if (ctx->len == AES_BLOCK_SIZE) {
> + aesti_encrypt(tctx, ctx->dg, ctx->dg);
> + ctx->len = 0;
> + }
> + }
> +
> + return 0;
> +}
> +
> +static int aesti_cbcmac_final(struct shash_desc *desc, u8 *out)
> +{
> + struct aes_ti_ctx *tctx = crypto_shash_ctx(desc->tfm);
> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(desc);
> +
> + if (ctx->len)
> + aesti_encrypt(tctx, out, ctx->dg);
> + else
> + memcpy(out, ctx->dg, AES_BLOCK_SIZE);
> +
> + return 0;
> +}
> +
> +static struct shash_alg cbcmac_alg = {
> + .base.cra_name = "cbcmac(aes)",
> + .base.cra_driver_name = "cbcmac-aes-ti",
> + .base.cra_priority = 100 + 1,
> + .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
> + .base.cra_blocksize = 1,
> + .base.cra_ctxsize = sizeof(struct aes_ti_ctx),
> + .base.cra_module = THIS_MODULE,
> +
> + .digestsize = AES_BLOCK_SIZE,
> + .init = aesti_cbcmac_init,
> + .update = aesti_cbcmac_update,
> + .final = aesti_cbcmac_final,
> + .setkey = aesti_cbcmac_setkey,
> + .descsize = sizeof(struct cbcmac_desc_ctx),
> +};
> +
> +static int __init aes_init(void)
> +{
> + int err;
> +
> + err = crypto_register_skcipher(&ctr_alg);
> + if (err)
> + return err;
> +
> + err = crypto_register_shash(&cbcmac_alg);
> + if (err)
> + crypto_unregister_skcipher(&ctr_alg);
> + return err;
> +}
> +
> +static void __exit aes_fini(void)
> +{
> + crypto_unregister_shash(&cbcmac_alg);
> + crypto_unregister_skcipher(&ctr_alg);
> +}
> +
> +module_init(aes_init);
> +module_exit(aes_fini);
> +
> +MODULE_DESCRIPTION("Generic time invariant AES transform in CTR and CBC-MAC modes");
> +MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
> +MODULE_LICENSE("GPL v2");
> +MODULE_ALIAS_CRYPTO("cbcmac(aes)");
> +MODULE_ALIAS_CRYPTO("ctr(aes)");
>
^ permalink raw reply
* Re: [RFC PATCH 4/4] crypto: aes - add generic time invariant AES for CTR/CCM/GCM
From: Ard Biesheuvel @ 2017-01-26 18:45 UTC (permalink / raw)
To: Krzysztof Kwiatkowski
Cc: linux-crypto@vger.kernel.org,
linux-arm-kernel@lists.infradead.org, Herbert Xu
In-Reply-To: <cd08e440-ce91-5ba4-d79c-d1995a3fa170@amongbytes.com>
On 26 January 2017 at 18:35, Krzysztof Kwiatkowski <kris@amongbytes.com> wrote:
> Ard,
>
> This is really interesting implementation. Is there a way to test if
> execution of this code is really constant time. Have you done any tests
> like that?
No, I haven't, and to be perfectly honest, I think it would only make
sense to do so on a loaded system, or the Sbox will be in the cache
all the time anyway.
> Adam Langley has proposed using modified version of valgrind
> (ctgrind) for that, but I wonder if you maybe thought about any
> alternative method?
>
I think it is quite feasible in the kernel to measure time spent in a
function each time it is invoked. I have never looked at ctgrind, but
if there is legitimate interest in this code, I will try to figure out
a way to find out how data dependent the latency of this algorithm is,
at least on hardware that I have access to.
>
> On 26/01/17 17:17, Ard Biesheuvel wrote:
>> Lookup table based AES is sensitive to timing attacks, which is
>> due to the fact that such table lookups are data dependent, and
>> the fact that 8 KB worth of tables covers a significant number of
>> cachelines on any architecture.
>>
>> For network facing algorithms such as CTR, CCM or GCM, this presents
>> a security risk, which is why arch specific AES ports are typically
>> time invariant, either through the use of special instructions, or
>> by using SIMD algorithms that don't rely on table lookups.
>>
>> For generic code, this is difficult to achieve without losing too
>> much performance, but we can improve the situation significantly by
>> switching to an implementation that only needs 256 bytes of table
>> data (the actual S-box itself), which can be prefetched at the start
>> of each block to eliminate data dependent latencies.
>>
>> Note that this only implements AES encryption, which is all we need
>> for CTR and CBC-MAC. AES decryption can easily be implemented in a
>> similar way, but is significantly more costly.
>>
>> This code runs at ~25 cycles per byte on ARM Cortex-A57 (while the
>> ordinary generic AES driver manages 18 cycles per byte on this
>> hardware).
>>
>> Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
>> ---
>> crypto/Kconfig | 14 +
>> crypto/Makefile | 1 +
>> crypto/aes_ti.c | 314 ++++++++++++++++++++
>> 3 files changed, 329 insertions(+)
>>
>> diff --git a/crypto/Kconfig b/crypto/Kconfig
>> index e8269d1b0282..ce1f6be9e48f 100644
>> --- a/crypto/Kconfig
>> +++ b/crypto/Kconfig
>> @@ -896,6 +896,20 @@ config CRYPTO_AES
>>
>> See <http://csrc.nist.gov/CryptoToolkit/aes/> for more information.
>>
>> +config CRYPTO_AES_TI
>> + tristate "Generic time invariant AES in CTR and CBC-MAC modes"
>> + select CRYPTO_BLKCIPHER
>> + select CRYPTO_HASH
>> + select CRYPTO_AES
>> + help
>> + This is a time invariant generic implementation of AES in CTR and
>> + CBC-MAC modes, intended for use by the generic CCM and GCM drivers,
>> + and other CTR based modes. Instead of using 8 lookup tables of 1 KB
>> + each, both for encryption and decryption, this implementation only
>> + uses a single S-box of 256 bytes, and attempts to eliminate data
>> + dependent latencies by prefetching the entire table into the cache
>> + at the start of each block.
>> +
>> config CRYPTO_AES_586
>> tristate "AES cipher algorithms (i586)"
>> depends on (X86 || UML_X86) && !64BIT
>> diff --git a/crypto/Makefile b/crypto/Makefile
>> index b8f0e3eb0791..bcd834536163 100644
>> --- a/crypto/Makefile
>> +++ b/crypto/Makefile
>> @@ -99,6 +99,7 @@ obj-$(CONFIG_CRYPTO_TWOFISH) += twofish_generic.o
>> obj-$(CONFIG_CRYPTO_TWOFISH_COMMON) += twofish_common.o
>> obj-$(CONFIG_CRYPTO_SERPENT) += serpent_generic.o
>> obj-$(CONFIG_CRYPTO_AES) += aes_generic.o
>> +obj-$(CONFIG_CRYPTO_AES_TI) += aes_ti.o
>> obj-$(CONFIG_CRYPTO_CAMELLIA) += camellia_generic.o
>> obj-$(CONFIG_CRYPTO_CAST_COMMON) += cast_common.o
>> obj-$(CONFIG_CRYPTO_CAST5) += cast5_generic.o
>> diff --git a/crypto/aes_ti.c b/crypto/aes_ti.c
>> new file mode 100644
>> index 000000000000..5ad80e063681
>> --- /dev/null
>> +++ b/crypto/aes_ti.c
>> @@ -0,0 +1,314 @@
>> +/*
>> + * Scalar (mostly) time invariant AES core transform for CTR/CCM/GCM
>> + *
>> + * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU General Public License version 2 as
>> + * published by the Free Software Foundation.
>> + */
>> +
>> +#include <crypto/aes.h>
>> +#include <crypto/internal/hash.h>
>> +#include <crypto/internal/skcipher.h>
>> +#include <linux/crypto.h>
>> +#include <linux/module.h>
>> +#include <asm/unaligned.h>
>> +
>> +struct aes_ti_ctx {
>> + u32 rk[AES_MAX_KEYLENGTH_U32];
>> + int rounds;
>> +};
>> +
>> +struct cbcmac_desc_ctx {
>> + unsigned int len;
>> + u8 dg[];
>> +};
>> +
>> +__weak const u8 __cacheline_aligned __aesti_sbox[] = {
>> + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5,
>> + 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
>> + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0,
>> + 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
>> + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc,
>> + 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
>> + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a,
>> + 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
>> + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0,
>> + 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
>> + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b,
>> + 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
>> + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85,
>> + 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
>> + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5,
>> + 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
>> + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17,
>> + 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
>> + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88,
>> + 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
>> + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c,
>> + 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
>> + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9,
>> + 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
>> + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6,
>> + 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
>> + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e,
>> + 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
>> + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94,
>> + 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
>> + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68,
>> + 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16,
>> +};
>> +
>> +static int aesti_set_key(struct aes_ti_ctx *ctx, const u8 *in_key,
>> + unsigned int key_len)
>> +{
>> + struct crypto_aes_ctx rk;
>> + int err;
>> +
>> + err = crypto_aes_expand_key(&rk, in_key, key_len);
>> + if (err)
>> + return err;
>> +
>> + memcpy(ctx->rk, rk.key_enc, sizeof(ctx->rk));
>> + ctx->rounds = 6 + key_len / 4;
>> +
>> + /*
>> + * In order to force the compiler to emit data independent Sbox lookups
>> + * at the start of each block, xor the first round key with values at
>> + * fixed indexes in the Sbox.
>> + */
>> + ctx->rk[0] ^= __aesti_sbox[ 0] ^ __aesti_sbox[128];
>> + ctx->rk[1] ^= __aesti_sbox[32] ^ __aesti_sbox[160];
>> + ctx->rk[2] ^= __aesti_sbox[64] ^ __aesti_sbox[192];
>> + ctx->rk[3] ^= __aesti_sbox[96] ^ __aesti_sbox[224];
>> +
>> + return 0;
>> +}
>> +
>> +static u32 mul_by_x(u32 w)
>> +{
>> + /* multiply by polynomial 'x' (0b10) in GF(2^8) */
>> + return ((w & 0x80808080) >> 7) * 0x1b ^ ((w & 0x7f7f7f7f) << 1);
>> +}
>> +
>> +static u32 mix_columns(u32 x)
>> +{
>> + u32 y = mul_by_x(x) ^ ror32(x, 16);
>> +
>> + return y ^ ror32(x ^ y, 8);
>> +}
>> +
>> +static __always_inline u32 subshift(u32 in[], int pos)
>> +{
>> + return (__aesti_sbox[in[pos] & 0xff]) ^
>> + (__aesti_sbox[(in[(pos + 1) % 4] >> 8) & 0xff] << 8) ^
>> + (__aesti_sbox[(in[(pos + 2) % 4] >> 16) & 0xff] << 16) ^
>> + (__aesti_sbox[(in[(pos + 3) % 4] >> 24) & 0xff] << 24);
>> +}
>> +
>> +static void aesti_encrypt(struct aes_ti_ctx *ctx, u8 *out, const u8 *in)
>> +{
>> + u32 st0[4], st1[4];
>> + u32 *rkp = ctx->rk + 4;
>> + int round;
>> +
>> + st0[0] = get_unaligned_le32(in);
>> + st0[1] = get_unaligned_le32(in + 4);
>> + st0[2] = get_unaligned_le32(in + 8);
>> + st0[3] = get_unaligned_le32(in + 12);
>> +
>> + st0[0] ^= __aesti_sbox[ 0] ^ __aesti_sbox[128] ^ ctx->rk[0];
>> + st0[1] ^= __aesti_sbox[32] ^ __aesti_sbox[160] ^ ctx->rk[1];
>> + st0[2] ^= __aesti_sbox[64] ^ __aesti_sbox[192] ^ ctx->rk[2];
>> + st0[3] ^= __aesti_sbox[96] ^ __aesti_sbox[224] ^ ctx->rk[3];
>> +
>> + for (round = 0;; round += 2) {
>> + st1[0] = mix_columns(subshift(st0, 0)) ^ *rkp++;
>> + st1[1] = mix_columns(subshift(st0, 1)) ^ *rkp++;
>> + st1[2] = mix_columns(subshift(st0, 2)) ^ *rkp++;
>> + st1[3] = mix_columns(subshift(st0, 3)) ^ *rkp++;
>> +
>> + if (round == ctx->rounds - 2)
>> + break;
>> +
>> + st0[0] = mix_columns(subshift(st1, 0)) ^ *rkp++;
>> + st0[1] = mix_columns(subshift(st1, 1)) ^ *rkp++;
>> + st0[2] = mix_columns(subshift(st1, 2)) ^ *rkp++;
>> + st0[3] = mix_columns(subshift(st1, 3)) ^ *rkp++;
>> + }
>> +
>> + put_unaligned_le32(subshift(st1, 0) ^ rkp[0], out);
>> + put_unaligned_le32(subshift(st1, 1) ^ rkp[1], out + 4);
>> + put_unaligned_le32(subshift(st1, 2) ^ rkp[2], out + 8);
>> + put_unaligned_le32(subshift(st1, 3) ^ rkp[3], out + 12);
>> +}
>> +
>> +static int aesti_ctr_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
>> + unsigned int key_len)
>> +{
>> + struct aes_ti_ctx *ctx = crypto_skcipher_ctx(tfm);
>> + int err;
>> +
>> + err = aesti_set_key(ctx, in_key, key_len);
>> + if (err)
>> + crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
>> + return err;
>> +}
>> +
>> +static int aesti_ctr_encrypt(struct skcipher_request *req)
>> +{
>> + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
>> + struct aes_ti_ctx *ctx = crypto_skcipher_ctx(tfm);
>> + struct skcipher_walk walk;
>> + u8 buf[AES_BLOCK_SIZE];
>> + int err;
>> +
>> + err = skcipher_walk_virt(&walk, req, true);
>> +
>> + while (walk.nbytes > 0) {
>> + u8 *dst = walk.dst.virt.addr;
>> + u8 *src = walk.src.virt.addr;
>> + int nbytes = walk.nbytes;
>> + int tail = 0;
>> +
>> + if (nbytes < walk.total) {
>> + nbytes = round_down(nbytes, AES_BLOCK_SIZE);
>> + tail = walk.nbytes % AES_BLOCK_SIZE;
>> + }
>> +
>> + do {
>> + int bsize = min(nbytes, AES_BLOCK_SIZE);
>> +
>> + aesti_encrypt(ctx, buf, walk.iv);
>> + if (dst != src)
>> + memcpy(dst, src, bsize);
>> + crypto_xor(dst, buf, bsize);
>> + crypto_inc(walk.iv, AES_BLOCK_SIZE);
>> +
>> + dst += AES_BLOCK_SIZE;
>> + src += AES_BLOCK_SIZE;
>> + nbytes -= AES_BLOCK_SIZE;
>> + } while (nbytes > 0);
>> +
>> + err = skcipher_walk_done(&walk, tail);
>> + }
>> + return err;
>> +}
>> +
>> +static struct skcipher_alg ctr_alg = {
>> + .base.cra_name = "ctr(aes)",
>> + .base.cra_driver_name = "ctr-aes-ti",
>> + .base.cra_priority = 100 + 1,
>> + .base.cra_blocksize = 1,
>> + .base.cra_ctxsize = sizeof(struct aes_ti_ctx),
>> + .base.cra_module = THIS_MODULE,
>> +
>> + .min_keysize = AES_MIN_KEY_SIZE,
>> + .max_keysize = AES_MAX_KEY_SIZE,
>> + .chunksize = AES_BLOCK_SIZE,
>> + .ivsize = AES_BLOCK_SIZE,
>> + .setkey = aesti_ctr_set_key,
>> + .encrypt = aesti_ctr_encrypt,
>> + .decrypt = aesti_ctr_encrypt,
>> +};
>> +
>> +static int aesti_cbcmac_setkey(struct crypto_shash *tfm,
>> + const u8 *in_key, unsigned int key_len)
>> +{
>> + struct aes_ti_ctx *ctx = crypto_shash_ctx(tfm);
>> + int err;
>> +
>> + err = aesti_set_key(ctx, in_key, key_len);
>> + if (err)
>> + crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
>> +
>> + return err;
>> +}
>> +
>> +static int aesti_cbcmac_init(struct shash_desc *desc)
>> +{
>> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(desc);
>> +
>> + memset(ctx->dg, 0, AES_BLOCK_SIZE);
>> + ctx->len = 0;
>> +
>> + return 0;
>> +}
>> +
>> +static int aesti_cbcmac_update(struct shash_desc *desc, const u8 *p,
>> + unsigned int len)
>> +{
>> + struct aes_ti_ctx *tctx = crypto_shash_ctx(desc->tfm);
>> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(desc);
>> +
>> + while (len--) {
>> + ctx->dg[ctx->len++] ^= *p++;
>> +
>> + if (ctx->len == AES_BLOCK_SIZE) {
>> + aesti_encrypt(tctx, ctx->dg, ctx->dg);
>> + ctx->len = 0;
>> + }
>> + }
>> +
>> + return 0;
>> +}
>> +
>> +static int aesti_cbcmac_final(struct shash_desc *desc, u8 *out)
>> +{
>> + struct aes_ti_ctx *tctx = crypto_shash_ctx(desc->tfm);
>> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(desc);
>> +
>> + if (ctx->len)
>> + aesti_encrypt(tctx, out, ctx->dg);
>> + else
>> + memcpy(out, ctx->dg, AES_BLOCK_SIZE);
>> +
>> + return 0;
>> +}
>> +
>> +static struct shash_alg cbcmac_alg = {
>> + .base.cra_name = "cbcmac(aes)",
>> + .base.cra_driver_name = "cbcmac-aes-ti",
>> + .base.cra_priority = 100 + 1,
>> + .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
>> + .base.cra_blocksize = 1,
>> + .base.cra_ctxsize = sizeof(struct aes_ti_ctx),
>> + .base.cra_module = THIS_MODULE,
>> +
>> + .digestsize = AES_BLOCK_SIZE,
>> + .init = aesti_cbcmac_init,
>> + .update = aesti_cbcmac_update,
>> + .final = aesti_cbcmac_final,
>> + .setkey = aesti_cbcmac_setkey,
>> + .descsize = sizeof(struct cbcmac_desc_ctx),
>> +};
>> +
>> +static int __init aes_init(void)
>> +{
>> + int err;
>> +
>> + err = crypto_register_skcipher(&ctr_alg);
>> + if (err)
>> + return err;
>> +
>> + err = crypto_register_shash(&cbcmac_alg);
>> + if (err)
>> + crypto_unregister_skcipher(&ctr_alg);
>> + return err;
>> +}
>> +
>> +static void __exit aes_fini(void)
>> +{
>> + crypto_unregister_shash(&cbcmac_alg);
>> + crypto_unregister_skcipher(&ctr_alg);
>> +}
>> +
>> +module_init(aes_init);
>> +module_exit(aes_fini);
>> +
>> +MODULE_DESCRIPTION("Generic time invariant AES transform in CTR and CBC-MAC modes");
>> +MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
>> +MODULE_LICENSE("GPL v2");
>> +MODULE_ALIAS_CRYPTO("cbcmac(aes)");
>> +MODULE_ALIAS_CRYPTO("ctr(aes)");
>>
>
^ permalink raw reply
* Re: [PATCH] crypto: camellia: add missing declarations
From: Nicholas Mc Guire @ 2017-01-27 5:31 UTC (permalink / raw)
To: Herbert Xu
Cc: Nicholas Mc Guire, David S. Miller, Thomas Gleixner, Ingo Molnar,
H. Peter Anvin, x86, linux-crypto, linux-kernel
In-Reply-To: <20170123140237.GB19957@gondor.apana.org.au>
On Mon, Jan 23, 2017 at 10:02:37PM +0800, Herbert Xu wrote:
> On Mon, Jan 16, 2017 at 05:06:51PM +0100, Nicholas Mc Guire wrote:
> > Add declarations for the camellia substitution box to allow a clean build.
> >
> > Signed-off-by: Nicholas Mc Guire <hofrat@osadl.org>
> > ---
> > Problem reported by sparse
> > arch/x86/crypto/camellia_glue.c:65:21: warning: symbol 'camellia_sp10011110' was not declared. Should it be static?
> > arch/x86/crypto/camellia_glue.c:154:21: warning: symbol 'camellia_sp22000222' was not declared. Should it be static?
> > arch/x86/crypto/camellia_glue.c:243:21: warning: symbol 'camellia_sp03303033' was not declared. Should it be static?
> > arch/x86/crypto/camellia_glue.c:332:21: warning: symbol 'camellia_sp00444404' was not declared. Should it be static?
> > arch/x86/crypto/camellia_glue.c:421:21: warning: symbol 'camellia_sp02220222' was not declared. Should it be static?
> > arch/x86/crypto/camellia_glue.c:510:21: warning: symbol 'camellia_sp30333033' was not declared. Should it be static?
> > arch/x86/crypto/camellia_glue.c:599:21: warning: symbol 'camellia_sp44044404' was not declared. Should it be static?
> > arch/x86/crypto/camellia_glue.c:688:21: warning: symbol 'camellia_sp11101110' was not declared. Should it be static?
> >
> > Patch was compile tested with: x86_64_defconfig +
> > CONFIG_CRYPTO_CAMELLIA_X86_64=m
> >
> > Patch is against 4.10-rc3 (localversion-next is next-20170116)
>
> This is arguably a sparse bug. These variables are only referenced
> by assembly code and already carries the __visible tag. So sparse
> should learn to suppress this warning when __visible is present.
I had assumed that __visible only would apply to LTO not to non-LTO
builds so the externally_visible attributed would not resolve this
and thus this warning seems correct.
Is this interpretation incorect ?
thx!
hofrat
^ permalink raw reply
* Re: [PATCH 2/4] crypto: ccm - switch to separate cbcmac driver
From: Ard Biesheuvel @ 2017-01-27 9:41 UTC (permalink / raw)
To: linux-crypto@vger.kernel.org,
linux-arm-kernel@lists.infradead.org
Cc: Herbert Xu, Ard Biesheuvel
In-Reply-To: <1485451063-11822-3-git-send-email-ard.biesheuvel@linaro.org>
On 26 January 2017 at 17:17, Ard Biesheuvel <ard.biesheuvel@linaro.org> wrote:
> Update the generic CCM driver to defer CBC-MAC processing to a
> dedicated CBC-MAC ahash transform rather than open coding this
> transform (and much of the associated scatterwalk plumbing) in
> the CCM driver itself.
>
> This cleans up the code considerably, but more importantly, it allows
> the use of alternative CBC-MAC implementations that don't suffer from
> performance degradation due to significant setup time (e.g., the NEON
> based AES code needs to load the entire S-box into SIMD registers, which
> cannot be amortized over the entire input when using the AES cipher
> directly)
>
> Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
> ---
> crypto/Kconfig | 1 +
> crypto/ccm.c | 373 +++++++++++++-------
> 2 files changed, 238 insertions(+), 136 deletions(-)
>
> diff --git a/crypto/Kconfig b/crypto/Kconfig
> index 160f08e721cc..e8269d1b0282 100644
> --- a/crypto/Kconfig
> +++ b/crypto/Kconfig
> @@ -263,6 +263,7 @@ comment "Authenticated Encryption with Associated Data"
> config CRYPTO_CCM
> tristate "CCM support"
> select CRYPTO_CTR
> + select CRYPTO_HASH
> select CRYPTO_AEAD
> help
> Support for Counter with CBC MAC. Required for IPsec.
> diff --git a/crypto/ccm.c b/crypto/ccm.c
> index 26b924d1e582..635f11fc52e7 100644
> --- a/crypto/ccm.c
> +++ b/crypto/ccm.c
> @@ -11,6 +11,7 @@
> */
>
> #include <crypto/internal/aead.h>
> +#include <crypto/internal/hash.h>
> #include <crypto/internal/skcipher.h>
> #include <crypto/scatterwalk.h>
> #include <linux/err.h>
> @@ -23,11 +24,11 @@
>
> struct ccm_instance_ctx {
> struct crypto_skcipher_spawn ctr;
> - struct crypto_spawn cipher;
> + struct crypto_ahash_spawn mac;
> };
>
> struct crypto_ccm_ctx {
> - struct crypto_cipher *cipher;
> + struct crypto_ahash *mac;
> struct crypto_skcipher *ctr;
> };
>
> @@ -44,7 +45,6 @@ struct crypto_rfc4309_req_ctx {
>
> struct crypto_ccm_req_priv_ctx {
> u8 odata[16];
> - u8 idata[16];
> u8 auth_tag[16];
> u32 ilen;
This is unused now.
> u32 flags;
> @@ -53,6 +53,15 @@ struct crypto_ccm_req_priv_ctx {
> struct skcipher_request skreq;
> };
>
> +struct cbcmac_tfm_ctx {
> + struct crypto_cipher *child;
> +};
> +
> +struct cbcmac_desc_ctx {
> + unsigned int len;
> + u8 dg[];
> +};
> +
> static inline struct crypto_ccm_req_priv_ctx *crypto_ccm_reqctx(
> struct aead_request *req)
> {
> @@ -84,7 +93,7 @@ static int crypto_ccm_setkey(struct crypto_aead *aead, const u8 *key,
> {
> struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
> struct crypto_skcipher *ctr = ctx->ctr;
> - struct crypto_cipher *tfm = ctx->cipher;
> + struct crypto_ahash *mac = ctx->mac;
> int err = 0;
>
> crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
> @@ -96,11 +105,11 @@ static int crypto_ccm_setkey(struct crypto_aead *aead, const u8 *key,
> if (err)
> goto out;
>
> - crypto_cipher_clear_flags(tfm, CRYPTO_TFM_REQ_MASK);
> - crypto_cipher_set_flags(tfm, crypto_aead_get_flags(aead) &
> + crypto_ahash_clear_flags(mac, CRYPTO_TFM_REQ_MASK);
> + crypto_ahash_set_flags(mac, crypto_aead_get_flags(aead) &
> CRYPTO_TFM_REQ_MASK);
> - err = crypto_cipher_setkey(tfm, key, keylen);
> - crypto_aead_set_flags(aead, crypto_cipher_get_flags(tfm) &
> + err = crypto_ahash_setkey(mac, key, keylen);
> + crypto_aead_set_flags(aead, crypto_ahash_get_flags(mac) &
> CRYPTO_TFM_RES_MASK);
>
> out:
> @@ -167,119 +176,59 @@ static int format_adata(u8 *adata, unsigned int a)
> return len;
> }
>
> -static void compute_mac(struct crypto_cipher *tfm, u8 *data, int n,
> - struct crypto_ccm_req_priv_ctx *pctx)
> -{
> - unsigned int bs = 16;
> - u8 *odata = pctx->odata;
> - u8 *idata = pctx->idata;
> - int datalen, getlen;
> -
> - datalen = n;
> -
> - /* first time in here, block may be partially filled. */
> - getlen = bs - pctx->ilen;
> - if (datalen >= getlen) {
> - memcpy(idata + pctx->ilen, data, getlen);
> - crypto_xor(odata, idata, bs);
> - crypto_cipher_encrypt_one(tfm, odata, odata);
> - datalen -= getlen;
> - data += getlen;
> - pctx->ilen = 0;
> - }
> -
> - /* now encrypt rest of data */
> - while (datalen >= bs) {
> - crypto_xor(odata, data, bs);
> - crypto_cipher_encrypt_one(tfm, odata, odata);
> -
> - datalen -= bs;
> - data += bs;
> - }
> -
> - /* check and see if there's leftover data that wasn't
> - * enough to fill a block.
> - */
> - if (datalen) {
> - memcpy(idata + pctx->ilen, data, datalen);
> - pctx->ilen += datalen;
> - }
> -}
> -
> -static void get_data_to_compute(struct crypto_cipher *tfm,
> - struct crypto_ccm_req_priv_ctx *pctx,
> - struct scatterlist *sg, unsigned int len)
> -{
> - struct scatter_walk walk;
> - u8 *data_src;
> - int n;
> -
> - scatterwalk_start(&walk, sg);
> -
> - while (len) {
> - n = scatterwalk_clamp(&walk, len);
> - if (!n) {
> - scatterwalk_start(&walk, sg_next(walk.sg));
> - n = scatterwalk_clamp(&walk, len);
> - }
> - data_src = scatterwalk_map(&walk);
> -
> - compute_mac(tfm, data_src, n, pctx);
> - len -= n;
> -
> - scatterwalk_unmap(data_src);
> - scatterwalk_advance(&walk, n);
> - scatterwalk_done(&walk, 0, len);
> - if (len)
> - crypto_yield(pctx->flags);
> - }
> -
> - /* any leftover needs padding and then encrypted */
> - if (pctx->ilen) {
> - int padlen;
> - u8 *odata = pctx->odata;
> - u8 *idata = pctx->idata;
> -
> - padlen = 16 - pctx->ilen;
> - memset(idata + pctx->ilen, 0, padlen);
> - crypto_xor(odata, idata, 16);
> - crypto_cipher_encrypt_one(tfm, odata, odata);
> - pctx->ilen = 0;
> - }
> -}
> -
> static int crypto_ccm_auth(struct aead_request *req, struct scatterlist *plain,
> unsigned int cryptlen)
> {
> + struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
> struct crypto_aead *aead = crypto_aead_reqtfm(req);
> struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
> - struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
> - struct crypto_cipher *cipher = ctx->cipher;
> + AHASH_REQUEST_ON_STACK(ahreq, ctx->mac);
> unsigned int assoclen = req->assoclen;
> - u8 *odata = pctx->odata;
> - u8 *idata = pctx->idata;
> - int err;
> + struct scatterlist sg[3];
> + u8 odata[16];
> + u8 idata[16];
> + int ilen, err;
>
> /* format control data for input */
> err = format_input(odata, req, cryptlen);
> if (err)
> goto out;
>
> - /* encrypt first block to use as start in computing mac */
> - crypto_cipher_encrypt_one(cipher, odata, odata);
> + sg_init_table(sg, 3);
> + sg_set_buf(&sg[0], odata, 16);
>
> /* format associated data and compute into mac */
> if (assoclen) {
> - pctx->ilen = format_adata(idata, assoclen);
> - get_data_to_compute(cipher, pctx, req->src, req->assoclen);
> + ilen = format_adata(idata, assoclen);
> + sg_set_buf(&sg[1], idata, ilen);
> + sg_chain(sg, 3, req->src);
> } else {
> - pctx->ilen = 0;
> + ilen = 0;
> + sg_chain(sg, 2, req->src);
> }
>
> - /* compute plaintext into mac */
> - if (cryptlen)
> - get_data_to_compute(cipher, pctx, plain, cryptlen);
> + ahash_request_set_tfm(ahreq, ctx->mac);
This needs
diff --git a/crypto/ccm.c b/crypto/ccm.c
index 635f11fc52e7..016059c0ffdc 100644
--- a/crypto/ccm.c
+++ b/crypto/ccm.c
@@ -208,6 +208,7 @@ static int crypto_ccm_auth(struct aead_request
*req, struct scatterlist *plain,
}
ahash_request_set_tfm(ahreq, ctx->mac);
+ ahash_request_set_callback(ahreq, pctx->flags, NULL, NULL);
ahash_request_set_crypt(ahreq, sg, NULL, assoclen + ilen + 16);
err = crypto_ahash_init(ahreq);
if (err)
here
> + ahash_request_set_crypt(ahreq, sg, NULL, assoclen + ilen + 16);
> + err = crypto_ahash_init(ahreq);
> + if (err)
> + goto out;
> + err = crypto_ahash_update(ahreq);
> + if (err)
> + goto out;
>
> + /* we need to pad the MAC input to a round multiple of the block size */
> + ilen = 16 - (assoclen + ilen) % 16;
> + if (ilen < 16) {
> + memset(idata, 0, ilen);
> + sg_init_table(sg, 2);
> + sg_set_buf(&sg[0], idata, ilen);
> + sg_chain(sg, 2, plain);
> + plain = sg;
> + cryptlen += ilen;
> + }
> +
> + ahash_request_set_crypt(ahreq, plain, pctx->odata, cryptlen);
> + err = crypto_ahash_finup(ahreq);
> out:
> return err;
> }
> @@ -453,21 +402,21 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
> struct aead_instance *inst = aead_alg_instance(tfm);
> struct ccm_instance_ctx *ictx = aead_instance_ctx(inst);
> struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm);
> - struct crypto_cipher *cipher;
> + struct crypto_ahash *mac;
> struct crypto_skcipher *ctr;
> unsigned long align;
> int err;
>
> - cipher = crypto_spawn_cipher(&ictx->cipher);
> - if (IS_ERR(cipher))
> - return PTR_ERR(cipher);
> + mac = crypto_spawn_ahash(&ictx->mac);
> + if (IS_ERR(mac))
> + return PTR_ERR(mac);
>
> ctr = crypto_spawn_skcipher(&ictx->ctr);
> err = PTR_ERR(ctr);
> if (IS_ERR(ctr))
> - goto err_free_cipher;
> + goto err_free_mac;
>
> - ctx->cipher = cipher;
> + ctx->mac = mac;
> ctx->ctr = ctr;
>
> align = crypto_aead_alignmask(tfm);
> @@ -479,8 +428,8 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
>
> return 0;
>
> -err_free_cipher:
> - crypto_free_cipher(cipher);
> +err_free_mac:
> + crypto_free_ahash(mac);
> return err;
> }
>
> @@ -488,7 +437,7 @@ static void crypto_ccm_exit_tfm(struct crypto_aead *tfm)
> {
> struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm);
>
> - crypto_free_cipher(ctx->cipher);
> + crypto_free_ahash(ctx->mac);
> crypto_free_skcipher(ctx->ctr);
> }
>
> @@ -496,7 +445,7 @@ static void crypto_ccm_free(struct aead_instance *inst)
> {
> struct ccm_instance_ctx *ctx = aead_instance_ctx(inst);
>
> - crypto_drop_spawn(&ctx->cipher);
> + crypto_drop_ahash(&ctx->mac);
> crypto_drop_skcipher(&ctx->ctr);
> kfree(inst);
> }
> @@ -505,12 +454,13 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> struct rtattr **tb,
> const char *full_name,
> const char *ctr_name,
> - const char *cipher_name)
> + const char *mac_name)
> {
> struct crypto_attr_type *algt;
> struct aead_instance *inst;
> struct skcipher_alg *ctr;
> - struct crypto_alg *cipher;
> + struct crypto_alg *mac_alg;
> + struct hash_alg_common *mac;
> struct ccm_instance_ctx *ictx;
> int err;
>
> @@ -521,25 +471,26 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
> return -EINVAL;
>
> - cipher = crypto_alg_mod_lookup(cipher_name, CRYPTO_ALG_TYPE_CIPHER,
> - CRYPTO_ALG_TYPE_MASK);
> - if (IS_ERR(cipher))
> - return PTR_ERR(cipher);
> + mac_alg = crypto_find_alg(mac_name, &crypto_ahash_type,
> + CRYPTO_ALG_TYPE_HASH,
> + CRYPTO_ALG_TYPE_AHASH_MASK |
> + CRYPTO_ALG_ASYNC);
> + if (IS_ERR(mac_alg))
> + return PTR_ERR(mac_alg);
>
> + mac = __crypto_hash_alg_common(mac_alg);
> err = -EINVAL;
> - if (cipher->cra_blocksize != 16)
> - goto out_put_cipher;
> + if (mac->digestsize != 16)
> + goto out_put_mac;
>
> inst = kzalloc(sizeof(*inst) + sizeof(*ictx), GFP_KERNEL);
> err = -ENOMEM;
> if (!inst)
> - goto out_put_cipher;
> + goto out_put_mac;
>
> ictx = aead_instance_ctx(inst);
> -
> - err = crypto_init_spawn(&ictx->cipher, cipher,
> - aead_crypto_instance(inst),
> - CRYPTO_ALG_TYPE_MASK);
> + err = crypto_init_ahash_spawn(&ictx->mac, mac,
> + aead_crypto_instance(inst));
> if (err)
> goto err_free_inst;
>
> @@ -548,7 +499,7 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> crypto_requires_sync(algt->type,
> algt->mask));
> if (err)
> - goto err_drop_cipher;
> + goto err_drop_mac;
>
> ctr = crypto_spawn_skcipher_alg(&ictx->ctr);
>
> @@ -564,16 +515,16 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> err = -ENAMETOOLONG;
> if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
> "ccm_base(%s,%s)", ctr->base.cra_driver_name,
> - cipher->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
> + mac->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
> goto err_drop_ctr;
>
> memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
>
> inst->alg.base.cra_flags = ctr->base.cra_flags & CRYPTO_ALG_ASYNC;
> - inst->alg.base.cra_priority = (cipher->cra_priority +
> + inst->alg.base.cra_priority = (mac->base.cra_priority +
> ctr->base.cra_priority) / 2;
> inst->alg.base.cra_blocksize = 1;
> - inst->alg.base.cra_alignmask = cipher->cra_alignmask |
> + inst->alg.base.cra_alignmask = mac->base.cra_alignmask |
> ctr->base.cra_alignmask |
> (__alignof__(u32) - 1);
> inst->alg.ivsize = 16;
> @@ -593,23 +544,24 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> if (err)
> goto err_drop_ctr;
>
> -out_put_cipher:
> - crypto_mod_put(cipher);
> +out_put_mac:
> + crypto_mod_put(mac_alg);
> return err;
>
> err_drop_ctr:
> crypto_drop_skcipher(&ictx->ctr);
> -err_drop_cipher:
> - crypto_drop_spawn(&ictx->cipher);
> +err_drop_mac:
> + crypto_drop_ahash(&ictx->mac);
> err_free_inst:
> kfree(inst);
> - goto out_put_cipher;
> + goto out_put_mac;
> }
>
> static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb)
> {
> const char *cipher_name;
> char ctr_name[CRYPTO_MAX_ALG_NAME];
> + char mac_name[CRYPTO_MAX_ALG_NAME];
> char full_name[CRYPTO_MAX_ALG_NAME];
>
> cipher_name = crypto_attr_alg_name(tb[1]);
> @@ -620,12 +572,16 @@ static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb)
> cipher_name) >= CRYPTO_MAX_ALG_NAME)
> return -ENAMETOOLONG;
>
> + if (snprintf(mac_name, CRYPTO_MAX_ALG_NAME, "cbcmac(%s)",
> + cipher_name) >= CRYPTO_MAX_ALG_NAME)
> + return -ENAMETOOLONG;
> +
> if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm(%s)", cipher_name) >=
> CRYPTO_MAX_ALG_NAME)
> return -ENAMETOOLONG;
>
> return crypto_ccm_create_common(tmpl, tb, full_name, ctr_name,
> - cipher_name);
> + mac_name);
> }
>
> static struct crypto_template crypto_ccm_tmpl = {
> @@ -899,14 +855,156 @@ static struct crypto_template crypto_rfc4309_tmpl = {
> .module = THIS_MODULE,
> };
>
> +static int crypto_cbcmac_digest_setkey(struct crypto_shash *parent,
> + const u8 *inkey, unsigned int keylen)
> +{
> + struct cbcmac_tfm_ctx *ctx = crypto_shash_ctx(parent);
> +
> + return crypto_cipher_setkey(ctx->child, inkey, keylen);
> +}
> +
> +static int crypto_cbcmac_digest_init(struct shash_desc *pdesc)
> +{
> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
> + int bs = crypto_shash_digestsize(pdesc->tfm);
> +
> + memset(ctx->dg, 0, bs);
> + ctx->len = 0;
> +
> + return 0;
> +}
> +
> +static int crypto_cbcmac_digest_update(struct shash_desc *pdesc, const u8 *p,
> + unsigned int len)
> +{
> + struct crypto_shash *parent = pdesc->tfm;
> + struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent);
> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
> + struct crypto_cipher *tfm = tctx->child;
> + int bs = crypto_shash_digestsize(parent);
> +
> + while (len--) {
> + ctx->dg[ctx->len++] ^= *p++;
> +
> + if (ctx->len == bs) {
> + crypto_cipher_encrypt_one(tfm, ctx->dg, ctx->dg);
> + ctx->len = 0;
> + }
> + }
> +
> + return 0;
> +}
> +
> +static int crypto_cbcmac_digest_final(struct shash_desc *pdesc, u8 *out)
> +{
> + struct crypto_shash *parent = pdesc->tfm;
> + struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent);
> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
> + struct crypto_cipher *tfm = tctx->child;
> + int bs = crypto_shash_digestsize(parent);
> +
> + if (ctx->len)
> + crypto_cipher_encrypt_one(tfm, out, ctx->dg);
> + else
> + memcpy(out, ctx->dg, bs);
> +
> + return 0;
> +}
> +
> +static int cbcmac_init_tfm(struct crypto_tfm *tfm)
> +{
> + struct crypto_cipher *cipher;
> + struct crypto_instance *inst = (void *)tfm->__crt_alg;
> + struct crypto_spawn *spawn = crypto_instance_ctx(inst);
> + struct cbcmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
> +
> + cipher = crypto_spawn_cipher(spawn);
> + if (IS_ERR(cipher))
> + return PTR_ERR(cipher);
> +
> + ctx->child = cipher;
> +
> + return 0;
> +};
> +
> +static void cbcmac_exit_tfm(struct crypto_tfm *tfm)
> +{
> + struct cbcmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
> + crypto_free_cipher(ctx->child);
> +}
> +
> +static int cbcmac_create(struct crypto_template *tmpl, struct rtattr **tb)
> +{
> + struct shash_instance *inst;
> + struct crypto_alg *alg;
> + int err;
> +
> + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
> + if (err)
> + return err;
> +
> + alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
> + CRYPTO_ALG_TYPE_MASK);
> + if (IS_ERR(alg))
> + return PTR_ERR(alg);
> +
> + inst = shash_alloc_instance("cbcmac", alg);
> + err = PTR_ERR(inst);
> + if (IS_ERR(inst))
> + goto out_put_alg;
> +
> + err = crypto_init_spawn(shash_instance_ctx(inst), alg,
> + shash_crypto_instance(inst),
> + CRYPTO_ALG_TYPE_MASK);
> + if (err)
> + goto out_free_inst;
> +
> + inst->alg.base.cra_priority = alg->cra_priority;
> + inst->alg.base.cra_blocksize = 1;
> +
> + inst->alg.digestsize = alg->cra_blocksize;
> + inst->alg.descsize = sizeof(struct cbcmac_desc_ctx) +
> + alg->cra_blocksize;
> +
> + inst->alg.base.cra_ctxsize = sizeof(struct cbcmac_tfm_ctx);
> + inst->alg.base.cra_init = cbcmac_init_tfm;
> + inst->alg.base.cra_exit = cbcmac_exit_tfm;
> +
> + inst->alg.init = crypto_cbcmac_digest_init;
> + inst->alg.update = crypto_cbcmac_digest_update;
> + inst->alg.final = crypto_cbcmac_digest_final;
> + inst->alg.setkey = crypto_cbcmac_digest_setkey;
> +
> + err = shash_register_instance(tmpl, inst);
> +
> +out_free_inst:
> + if (err)
> + shash_free_instance(shash_crypto_instance(inst));
> +
> +out_put_alg:
> + crypto_mod_put(alg);
> + return err;
> +}
> +
> +static struct crypto_template crypto_cbcmac_tmpl = {
> + .name = "cbcmac",
> + .create = cbcmac_create,
> + .free = shash_free_instance,
> + .module = THIS_MODULE,
> +};
> +
> static int __init crypto_ccm_module_init(void)
> {
> int err;
>
> - err = crypto_register_template(&crypto_ccm_base_tmpl);
> + err = crypto_register_template(&crypto_cbcmac_tmpl);
> if (err)
> goto out;
>
> + err = crypto_register_template(&crypto_ccm_base_tmpl);
> + if (err)
> + goto out_undo_cbcmac;
> +
> err = crypto_register_template(&crypto_ccm_tmpl);
> if (err)
> goto out_undo_base;
> @@ -922,6 +1020,8 @@ static int __init crypto_ccm_module_init(void)
> crypto_unregister_template(&crypto_ccm_tmpl);
> out_undo_base:
> crypto_unregister_template(&crypto_ccm_base_tmpl);
> +out_undo_cbcmac:
> + crypto_register_template(&crypto_cbcmac_tmpl);
> goto out;
> }
>
> @@ -930,6 +1030,7 @@ static void __exit crypto_ccm_module_exit(void)
> crypto_unregister_template(&crypto_rfc4309_tmpl);
> crypto_unregister_template(&crypto_ccm_tmpl);
> crypto_unregister_template(&crypto_ccm_base_tmpl);
> + crypto_unregister_template(&crypto_cbcmac_tmpl);
> }
>
> module_init(crypto_ccm_module_init);
> --
> 2.7.4
>
^ permalink raw reply related
* [PATCH 0/8] Bug fixes
From: Harsh Jain @ 2017-01-27 10:39 UTC (permalink / raw)
To: herbert, linux-crypto, hariprasad, netdev, arjun, atul.gupta; +Cc: Harsh Jain
This patch series is based on Herbert's cryptodev-2.6 tree and depends on
patch series "Bug Fixes for 4.10". It includes Bug Fixes.
Atul Gupta (2)
crypto:chcr-Change flow IDs
crypto:chcr- Fix wrong typecasting
Harsh Jain (8):
crypto:chcr- Fix key length for RFC4106
crypto:chcr-fix itnull.cocci warnings
crypto:chcr- Use cipher instead of Block Cipher in gcm setkey
crypto:chcr: Change cra_flags for cipher algos
crypto:chcr- Change algo priority
crypto:chcr-Fix Smatch Complaint
drivers/crypto/chelsio/chcr_algo.c | 53 ++++++++++++++-------------
drivers/crypto/chelsio/chcr_algo.h | 9 +++--
drivers/crypto/chelsio/chcr_core.c | 11 +++---
drivers/crypto/chelsio/chcr_core.h | 1 +
drivers/crypto/chelsio/chcr_crypto.h | 2 +-
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 8 ++++
6 files changed, 47 insertions(+), 37 deletions(-)
mode change 100644 => 100755 drivers/crypto/chelsio/chcr_algo.c
--
1.8.2.3
^ permalink raw reply
* [PATCH 2/8] crypto:chcr- Fix key length for RFC4106
From: Harsh Jain @ 2017-01-27 10:39 UTC (permalink / raw)
To: herbert, linux-crypto, hariprasad, netdev, arjun, atul.gupta; +Cc: Harsh Jain
In-Reply-To: <cover.1485501428.git.harsh@chelsio.com>
Check keylen before copying salt to avoid wrap around of Integer.
Signed-off-by: Harsh Jain <harsh@chelsio.com>
---
drivers/crypto/chelsio/chcr_algo.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index deec7c0..6c2dea3 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -2194,8 +2194,8 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int ck_size;
int ret = 0, key_ctx_size = 0;
- if (get_aead_subtype(aead) ==
- CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
+ if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
+ keylen > 3) {
keylen -= 4; /* nonce/salt is present in the last 4 bytes */
memcpy(aeadctx->salt, key + keylen, 4);
}
--
1.8.2.3
^ permalink raw reply related
* [PATCH 3/8] crypto:chcr-fix itnull.cocci warnings
From: Harsh Jain @ 2017-01-27 10:39 UTC (permalink / raw)
To: herbert, linux-crypto, hariprasad, netdev, arjun, atul.gupta
Cc: Harsh Jain, Julia Lawall, Fengguang Wu
In-Reply-To: <cover.1485501428.git.harsh@chelsio.com>
The first argument to list_for_each_entry cannot be NULL.
Generated by: scripts/coccinelle/iterators/itnull.cocci
Signed-off-by: Julia Lawall <julia.lawall@lip6.fr>
Signed-off-by: Fengguang Wu <fengguang.wu@intel.com>
Signed-off-by: Harsh Jain <harsh@chelsio.com>
---
drivers/crypto/chelsio/chcr_core.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c
index 1c65f07..2bfd61a 100644
--- a/drivers/crypto/chelsio/chcr_core.c
+++ b/drivers/crypto/chelsio/chcr_core.c
@@ -61,7 +61,7 @@ int assign_chcr_device(struct chcr_dev **dev)
*/
mutex_lock(&dev_mutex); /* TODO ? */
list_for_each_entry(u_ctx, &uld_ctx_list, entry)
- if (u_ctx && u_ctx->dev) {
+ if (u_ctx->dev) {
*dev = u_ctx->dev;
ret = 0;
break;
--
1.8.2.3
^ permalink raw reply related
* [PATCH 4/8] crypto:chcr- Use cipher instead of Block Cipher in gcm setkey
From: Harsh Jain @ 2017-01-27 10:39 UTC (permalink / raw)
To: herbert, linux-crypto, hariprasad, netdev, arjun, atul.gupta; +Cc: Harsh Jain
In-Reply-To: <cover.1485501428.git.harsh@chelsio.com>
1 Block of encrption can be done with aes-generic. no need of
cbc(aes). This patch replaces cbc(aes-generic) with aes-generic.
Signed-off-by: Harsh Jain <harsh@chelsio.com>
---
drivers/crypto/chelsio/chcr_algo.c | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index 6c2dea3..d335943 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -2189,8 +2189,7 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
struct chcr_context *ctx = crypto_aead_ctx(aead);
struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
- struct blkcipher_desc h_desc;
- struct scatterlist src[1];
+ struct crypto_cipher *cipher;
unsigned int ck_size;
int ret = 0, key_ctx_size = 0;
@@ -2223,27 +2222,26 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
CHCR_KEYCTX_MAC_KEY_SIZE_128,
0, 0,
key_ctx_size >> 4);
- /* Calculate the H = CIPH(K, 0 repeated 16 times) using sync aes
- * blkcipher It will go on key context
+ /* Calculate the H = CIPH(K, 0 repeated 16 times).
+ * It will go in key context
*/
- h_desc.tfm = crypto_alloc_blkcipher("cbc(aes-generic)", 0, 0);
- if (IS_ERR(h_desc.tfm)) {
+ cipher = crypto_alloc_cipher("aes-generic", 0, 0);
+ if (IS_ERR(cipher)) {
aeadctx->enckey_len = 0;
ret = -ENOMEM;
goto out;
}
- h_desc.flags = 0;
- ret = crypto_blkcipher_setkey(h_desc.tfm, key, keylen);
+
+ ret = crypto_cipher_setkey(cipher, key, keylen);
if (ret) {
aeadctx->enckey_len = 0;
goto out1;
}
memset(gctx->ghash_h, 0, AEAD_H_SIZE);
- sg_init_one(&src[0], gctx->ghash_h, AEAD_H_SIZE);
- ret = crypto_blkcipher_encrypt(&h_desc, &src[0], &src[0], AEAD_H_SIZE);
+ crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h);
out1:
- crypto_free_blkcipher(h_desc.tfm);
+ crypto_free_cipher(cipher);
out:
return ret;
}
--
1.8.2.3
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox