* [PATCH 0/7]crypto:chelsio: Bugs fixes
@ 2017-10-03 6:46 Harsh Jain
2017-10-03 6:46 ` [PATCH 1/7] crypto:chelsio: Remove unused parameter Harsh Jain
` (7 more replies)
0 siblings, 8 replies; 18+ messages in thread
From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw)
To: herbert, linux-crypto, netdev; +Cc: Harsh Jain
It includes bug fix and performance improvement changes.
Harsh Jain (7):
crypto:gf128mul: The x8_ble multiplication functions
crypto:chelsio:Use x8_ble gf multiplication to calculate IV.
crypto:chelsio:Remove allocation of sg list to implement 2K limit of
dsgl header
crypto:chelsio:Move DMA un/mapping to chcr from lld cxgb4 driver
crypto:chelsio: Fix memory leak
crypto:chelsio: Remove unused parameter
crypto:chelsio: Check error code with IS_ERR macro
crypto/gf128mul.c | 13 +
drivers/crypto/chelsio/chcr_algo.c | 1784 +++++++++++++++++-------------
drivers/crypto/chelsio/chcr_algo.h | 57 +-
drivers/crypto/chelsio/chcr_core.c | 8 +-
drivers/crypto/chelsio/chcr_core.h | 2 +-
drivers/crypto/chelsio/chcr_crypto.h | 121 +-
drivers/net/ethernet/chelsio/cxgb4/sge.c | 8 +-
include/crypto/gf128mul.h | 2 +-
8 files changed, 1166 insertions(+), 829 deletions(-)
--
2.1.4
^ permalink raw reply [flat|nested] 18+ messages in thread* [PATCH 1/7] crypto:chelsio: Remove unused parameter 2017-10-03 6:46 [PATCH 0/7]crypto:chelsio: Bugs fixes Harsh Jain @ 2017-10-03 6:46 ` Harsh Jain 2017-10-03 6:46 ` [PATCH 2/7] crypto:chelsio: Check error code with IS_ERR macro Harsh Jain ` (6 subsequent siblings) 7 siblings, 0 replies; 18+ messages in thread From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw) To: herbert, linux-crypto, netdev; +Cc: Harsh Jain Remove unused parameter sent to latest fw. Signed-off-by: Harsh Jain <harsh@chelsio.com> --- drivers/crypto/chelsio/chcr_algo.c | 43 +++++++++++++++----------------------- drivers/crypto/chelsio/chcr_algo.h | 12 +++++------ 2 files changed, 23 insertions(+), 32 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 0e81607..bdb1014 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -577,36 +577,27 @@ static int chcr_cipher_fallback(struct crypto_skcipher *cipher, static inline void create_wreq(struct chcr_context *ctx, struct chcr_wr *chcr_req, void *req, struct sk_buff *skb, - int kctx_len, int hash_sz, - int is_iv, + int hash_sz, unsigned int sc_len, unsigned int lcb) { struct uld_ctx *u_ctx = ULD_CTX(ctx); - int iv_loc = IV_DSGL; int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; - unsigned int immdatalen = 0, nr_frags = 0; + unsigned int immdatalen = 0; - if (is_ofld_imm(skb)) { + if (is_ofld_imm(skb)) immdatalen = skb->data_len; - iv_loc = IV_IMMEDIATE; - } else { - nr_frags = skb_shinfo(skb)->nr_frags; - } - chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen, - ((sizeof(chcr_req->key_ctx) + kctx_len) >> 4)); + chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE; chcr_req->wreq.pld_size_hash_size = - htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) | - FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); + htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); chcr_req->wreq.len16_pkd = htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP( (calc_tx_flits_ofld(skb) * 8), 16))); chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid, - is_iv ? iv_loc : IV_NOP, !!lcb, - ctx->tx_qidx); + !!lcb, ctx->tx_qidx); chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id, qid); @@ -616,7 +607,7 @@ static inline void create_wreq(struct chcr_context *ctx, chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen); chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + sizeof(chcr_req->key_ctx) + - kctx_len + sc_len + immdatalen); + sc_len + immdatalen); } /** @@ -706,8 +697,8 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes); atomic_inc(&adap->chcr_stats.cipher_rqst); - create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, kctx_len, 0, 1, - sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl, + create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, 0, + sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl + kctx_len, ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); reqctx->skb = skb; skb_get(skb); @@ -1417,8 +1408,8 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, if (param->sg_len != 0) write_sg_to_skb(skb, &frags, req->src, param->sg_len); atomic_inc(&adap->chcr_stats.digest_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, - hash_size_in_response, 0, DUMMY_BYTES, 0); + create_wreq(ctx, chcr_req, &req->base, skb, hash_size_in_response, + DUMMY_BYTES + kctx_len, 0); req_ctx->skb = skb; skb_get(skb); return skb; @@ -2080,8 +2071,8 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, write_buffer_to_skb(skb, &frags, req->iv, ivsize); write_sg_to_skb(skb, &frags, src, req->cryptlen); atomic_inc(&adap->chcr_stats.cipher_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); + create_wreq(ctx, chcr_req, &req->base, skb, size, + sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0); reqctx->skb = skb; skb_get(skb); @@ -2396,8 +2387,8 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, skb_set_transport_header(skb, transhdr_len); frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx); atomic_inc(&adap->chcr_stats.aead_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, 0, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); + create_wreq(ctx, chcr_req, &req->base, skb, 0, + sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0); reqctx->skb = skb; skb_get(skb); return skb; @@ -2554,8 +2545,8 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); write_sg_to_skb(skb, &frags, src, req->cryptlen); atomic_inc(&adap->chcr_stats.aead_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size, + create_wreq(ctx, chcr_req, &req->base, skb, size, + sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, reqctx->verify); reqctx->skb = skb; skb_get(skb); diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index 583008d..54851ec 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -176,21 +176,21 @@ KEY_CONTEXT_SALT_PRESENT_V(1) | \ KEY_CONTEXT_CTX_LEN_V((ctx_len))) -#define FILL_WR_OP_CCTX_SIZE(len, ctx_len) \ +#define FILL_WR_OP_CCTX_SIZE \ htonl( \ FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \ FW_CRYPTO_LOOKASIDE_WR) | \ FW_CRYPTO_LOOKASIDE_WR_COMPL_V(0) | \ - FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V((len)) | \ - FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(1) | \ - FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V((ctx_len))) + FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V((0)) | \ + FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(0) | \ + FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V(0)) -#define FILL_WR_RX_Q_ID(cid, qid, wr_iv, lcb, fid) \ +#define FILL_WR_RX_Q_ID(cid, qid, lcb, fid) \ htonl( \ FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V((cid)) | \ FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V((qid)) | \ FW_CRYPTO_LOOKASIDE_WR_LCB_V((lcb)) | \ - FW_CRYPTO_LOOKASIDE_WR_IV_V((wr_iv)) | \ + FW_CRYPTO_LOOKASIDE_WR_IV_V((IV_NOP)) | \ FW_CRYPTO_LOOKASIDE_WR_FQIDX_V(fid)) #define FILL_ULPTX_CMD_DEST(cid, qid) \ -- 2.1.4 ^ permalink raw reply related [flat|nested] 18+ messages in thread
* [PATCH 2/7] crypto:chelsio: Check error code with IS_ERR macro 2017-10-03 6:46 [PATCH 0/7]crypto:chelsio: Bugs fixes Harsh Jain 2017-10-03 6:46 ` [PATCH 1/7] crypto:chelsio: Remove unused parameter Harsh Jain @ 2017-10-03 6:46 ` Harsh Jain 2017-10-03 6:46 ` [PATCH 3/7] crypto:gf128mul: The x8_ble multiplication functions Harsh Jain ` (5 subsequent siblings) 7 siblings, 0 replies; 18+ messages in thread From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw) To: herbert, linux-crypto, netdev; +Cc: Harsh Jain Check and return proper error code. Signed-off-by: Harsh Jain <harsh@chelsio.com> --- drivers/crypto/chelsio/chcr_algo.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index bdb1014..e4bf32d 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -1455,8 +1455,8 @@ static int chcr_ahash_update(struct ahash_request *req) req_ctx->result = 0; req_ctx->data_len += params.sg_len + params.bfr_len; skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; + if (IS_ERR(skb)) + return PTR_ERR(skb); if (remainder) { u8 *temp; @@ -1519,8 +1519,8 @@ static int chcr_ahash_final(struct ahash_request *req) params.more = 0; } skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; + if (IS_ERR(skb)) + return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); @@ -1570,8 +1570,8 @@ static int chcr_ahash_finup(struct ahash_request *req) } skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; + if (IS_ERR(skb)) + return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); @@ -1621,8 +1621,8 @@ static int chcr_ahash_digest(struct ahash_request *req) } skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; + if (IS_ERR(skb)) + return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); -- 2.1.4 ^ permalink raw reply related [flat|nested] 18+ messages in thread
* [PATCH 3/7] crypto:gf128mul: The x8_ble multiplication functions 2017-10-03 6:46 [PATCH 0/7]crypto:chelsio: Bugs fixes Harsh Jain 2017-10-03 6:46 ` [PATCH 1/7] crypto:chelsio: Remove unused parameter Harsh Jain 2017-10-03 6:46 ` [PATCH 2/7] crypto:chelsio: Check error code with IS_ERR macro Harsh Jain @ 2017-10-03 6:46 ` Harsh Jain 2017-10-03 14:58 ` David Laight 2017-10-03 6:46 ` [PATCH 4/7] crypto:chelsio:Use x8_ble gf multiplication to calculate IV Harsh Jain ` (4 subsequent siblings) 7 siblings, 1 reply; 18+ messages in thread From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw) To: herbert, linux-crypto, netdev; +Cc: Harsh Jain It multiply GF(2^128) elements in the ble format. It will be used by chelsio driver to fasten gf multiplication. Signed-off-by: Harsh Jain <harsh@chelsio.com> --- crypto/gf128mul.c | 13 +++++++++++++ include/crypto/gf128mul.h | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c index dc01212..24e6019 100644 --- a/crypto/gf128mul.c +++ b/crypto/gf128mul.c @@ -156,6 +156,19 @@ static void gf128mul_x8_bbe(be128 *x) x->b = cpu_to_be64((b << 8) ^ _tt); } +void gf128mul_x8_ble(le128 *r, const le128 *x) +{ + u64 a = le64_to_cpu(x->a); + u64 b = le64_to_cpu(x->b); + + /* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */ + u64 _tt = gf128mul_table_be[a >> 56]; + + r->a = cpu_to_le64((a << 8) | (b >> 56)); + r->b = cpu_to_le64((b << 8) ^ _tt); +} +EXPORT_SYMBOL(gf128mul_x8_ble); + void gf128mul_lle(be128 *r, const be128 *b) { be128 p[8]; diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h index 0977fb1..fa0a63d 100644 --- a/include/crypto/gf128mul.h +++ b/include/crypto/gf128mul.h @@ -227,7 +227,7 @@ struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g); struct gf128mul_4k *gf128mul_init_4k_bbe(const be128 *g); void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t); void gf128mul_4k_bbe(be128 *a, const struct gf128mul_4k *t); - +void gf128mul_x8_ble(le128 *r, const le128 *x); static inline void gf128mul_free_4k(struct gf128mul_4k *t) { kzfree(t); -- 2.1.4 ^ permalink raw reply related [flat|nested] 18+ messages in thread
* RE: [PATCH 3/7] crypto:gf128mul: The x8_ble multiplication functions 2017-10-03 6:46 ` [PATCH 3/7] crypto:gf128mul: The x8_ble multiplication functions Harsh Jain @ 2017-10-03 14:58 ` David Laight 2017-10-05 11:20 ` Harsh Jain 0 siblings, 1 reply; 18+ messages in thread From: David Laight @ 2017-10-03 14:58 UTC (permalink / raw) To: 'Harsh Jain', herbert@gondor.apana.org.au, linux-crypto@vger.kernel.org, netdev@vger.kernel.org From: Harsh Jain > Sent: 03 October 2017 07:46 > It multiply GF(2^128) elements in the ble format. > It will be used by chelsio driver to fasten gf multiplication. ^ speed up ?? David ^ permalink raw reply [flat|nested] 18+ messages in thread
* Re: [PATCH 3/7] crypto:gf128mul: The x8_ble multiplication functions 2017-10-03 14:58 ` David Laight @ 2017-10-05 11:20 ` Harsh Jain 0 siblings, 0 replies; 18+ messages in thread From: Harsh Jain @ 2017-10-05 11:20 UTC (permalink / raw) To: David Laight, herbert@gondor.apana.org.au, linux-crypto@vger.kernel.org, netdev@vger.kernel.org On 03-10-2017 20:28, David Laight wrote: > From: Harsh Jain >> Sent: 03 October 2017 07:46 >> It multiply GF(2^128) elements in the ble format. >> It will be used by chelsio driver to fasten gf multiplication. > ^ speed up ?? It should be speed up. Will fix the same in V2. Thanks > > David > ^ permalink raw reply [flat|nested] 18+ messages in thread
* [PATCH 4/7] crypto:chelsio:Use x8_ble gf multiplication to calculate IV. 2017-10-03 6:46 [PATCH 0/7]crypto:chelsio: Bugs fixes Harsh Jain ` (2 preceding siblings ...) 2017-10-03 6:46 ` [PATCH 3/7] crypto:gf128mul: The x8_ble multiplication functions Harsh Jain @ 2017-10-03 6:46 ` Harsh Jain 2017-10-03 6:46 ` [PATCH 5/7] crypto:chelsio:Remove allocation of sg list to implement 2K limit of dsgl header Harsh Jain ` (3 subsequent siblings) 7 siblings, 0 replies; 18+ messages in thread From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw) To: herbert, linux-crypto, netdev; +Cc: Harsh Jain gf128mul_x8_ble() will reduce gf Multiplication iteration by 8. Signed-off-by: Harsh Jain <harsh@chelsio.com> --- drivers/crypto/chelsio/chcr_algo.c | 11 +++++++++-- drivers/crypto/chelsio/chcr_crypto.h | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index e4bf32d..e0ab34a 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -888,9 +888,11 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) int ret, i; u8 *key; unsigned int keylen; + int round = reqctx->last_req_len / AES_BLOCK_SIZE; + int round8 = round / 8; cipher = ablkctx->aes_generic; - memcpy(iv, req->info, AES_BLOCK_SIZE); + memcpy(iv, reqctx->iv, AES_BLOCK_SIZE); keylen = ablkctx->enckey_len / 2; key = ablkctx->key + keylen; @@ -899,7 +901,10 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) goto out; crypto_cipher_encrypt_one(cipher, iv, iv); - for (i = 0; i < (reqctx->processed / AES_BLOCK_SIZE); i++) + for (i = 0; i < round8; i++) + gf128mul_x8_ble((le128 *)iv, (le128 *)iv); + + for (i = 0; i < (round % 8); i++) gf128mul_x_ble((le128 *)iv, (le128 *)iv); crypto_cipher_decrypt_one(cipher, iv, iv); @@ -1040,6 +1045,7 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, CRYPTO_ALG_SUB_TYPE_CTR) bytes = adjust_ctr_overflow(reqctx->iv, bytes); reqctx->processed += bytes; + reqctx->last_req_len = bytes; wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; wrparam.req = req; wrparam.bytes = bytes; @@ -1132,6 +1138,7 @@ static int process_cipher(struct ablkcipher_request *req, goto error; } reqctx->processed = bytes; + reqctx->last_req_len = bytes; reqctx->dst = reqctx->dstsg; reqctx->op = op_type; wrparam.qid = qid; diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 30af1ee..b3722b3 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -247,6 +247,7 @@ struct chcr_blkcipher_req_ctx { struct scatterlist *dst; struct scatterlist *newdstsg; unsigned int processed; + unsigned int last_req_len; unsigned int op; short int dst_nents; u8 iv[CHCR_MAX_CRYPTO_IV_LEN]; -- 2.1.4 ^ permalink raw reply related [flat|nested] 18+ messages in thread
* [PATCH 5/7] crypto:chelsio:Remove allocation of sg list to implement 2K limit of dsgl header 2017-10-03 6:46 [PATCH 0/7]crypto:chelsio: Bugs fixes Harsh Jain ` (3 preceding siblings ...) 2017-10-03 6:46 ` [PATCH 4/7] crypto:chelsio:Use x8_ble gf multiplication to calculate IV Harsh Jain @ 2017-10-03 6:46 ` Harsh Jain 2017-10-03 6:46 ` [PATCH 6/7] crypto:chelsio:Move DMA un/mapping to chcr from lld cxgb4 driver Harsh Jain ` (2 subsequent siblings) 7 siblings, 0 replies; 18+ messages in thread From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw) To: herbert, linux-crypto, netdev; +Cc: Harsh Jain Update DMA address index instead of allocating new sg list to impose 2k size limit for each entry. Signed-off-by: Harsh Jain <harsh@chelsio.com> --- drivers/crypto/chelsio/chcr_algo.c | 237 +++++++++++------------------------ drivers/crypto/chelsio/chcr_algo.h | 3 +- drivers/crypto/chelsio/chcr_core.h | 2 +- drivers/crypto/chelsio/chcr_crypto.h | 6 - 4 files changed, 76 insertions(+), 172 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index e0ab34a..b13991d 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -117,6 +117,21 @@ static inline unsigned int sgl_len(unsigned int n) return (3 * n) / 2 + (n & 1) + 2; } +static int dstsg_2k(struct scatterlist *sgl, unsigned int reqlen) +{ + int nents = 0; + unsigned int less; + + while (sgl && reqlen) { + less = min(reqlen, sgl->length); + nents += DIV_ROUND_UP(less, CHCR_SG_SIZE); + reqlen -= less; + sgl = sg_next(sgl); + } + + return nents; +} + static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) { u8 temp[SHA512_DIGEST_SIZE]; @@ -166,8 +181,6 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, kfree_skb(ctx_req.ctx.reqctx->skb); ctx_req.ctx.reqctx->skb = NULL; } - free_new_sg(ctx_req.ctx.reqctx->newdstsg); - ctx_req.ctx.reqctx->newdstsg = NULL; if (ctx_req.ctx.reqctx->verify == VERIFY_SW) { chcr_verify_tag(ctx_req.req.aead_req, input, &err); @@ -388,31 +401,41 @@ static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl, { struct phys_sge_pairs *to; unsigned int len = 0, left_size = sg_param->obsize; - unsigned int nents = sg_param->nents, i, j = 0; + unsigned int j = 0; + int offset, ent_len; phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); + to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl + + sizeof(struct cpl_rx_phys_dsgl)); + while (left_size && sg) { + len = min_t(u32, left_size, sg_dma_len(sg)); + offset = 0; + while (len) { + ent_len = min_t(u32, len, CHCR_SG_SIZE); + to->len[j % 8] = htons(ent_len); + to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + + offset); + offset += ent_len; + len -= ent_len; + j++; + if ((j % 8) == 0) + to++; + } + left_size -= min(left_size, sg_dma_len(sg)); + sg = sg_next(sg); + } phys_cpl->pcirlxorder_to_noofsgentr = htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | CPL_RX_PHYS_DSGL_DCAID_V(0) | - CPL_RX_PHYS_DSGL_NOOFSGENTR_V(nents)); + CPL_RX_PHYS_DSGL_NOOFSGENTR_V(j)); phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; phys_cpl->rss_hdr_int.qid = htons(sg_param->qid); phys_cpl->rss_hdr_int.hash_val = 0; - to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl + - sizeof(struct cpl_rx_phys_dsgl)); - for (i = 0; nents && left_size; to++) { - for (j = 0; j < 8 && nents && left_size; j++, nents--) { - len = min(left_size, sg_dma_len(sg)); - to->len[j] = htons(len); - to->addr[j] = cpu_to_be64(sg_dma_address(sg)); - left_size -= len; - sg = sg_next(sg); - } - } + } static inline int map_writesg_phys_cpl(struct device *dev, @@ -523,31 +546,33 @@ static int generate_copy_rrkey(struct ablk_ctx *ablkctx, static int chcr_sg_ent_in_wr(struct scatterlist *src, struct scatterlist *dst, unsigned int minsg, - unsigned int space, - short int *sent, - short int *dent) + unsigned int space) { int srclen = 0, dstlen = 0; int srcsg = minsg, dstsg = 0; + int offset = 0, less; - *sent = 0; - *dent = 0; while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) && space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { srclen += src->length; srcsg++; + offset = 0; while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { if (srclen <= dstlen) break; - dstlen += dst->length; - dst = sg_next(dst); + less = min_t(unsigned int, dst->length - offset, + CHCR_SG_SIZE); + dstlen += less; + offset += less; + if (offset == dst->length) { + dst = sg_next(dst); + offset = 0; + } dstsg++; } src = sg_next(src); } - *sent = srcsg - minsg; - *dent = dstsg; return min(srclen, dstlen); } @@ -631,13 +656,15 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) struct phys_sge_parm sg_param; unsigned int frags = 0, transhdr_len, phys_dsgl; int error; + int nents; unsigned int ivsize = AES_BLOCK_SIZE, kctx_len; gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; struct adapter *adap = padap(ctx->dev); - phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents); - + reqctx->dst_nents = sg_nents_for_len(reqctx->dst, wrparam->bytes); + nents = dstsg_2k(reqctx->dst, wrparam->bytes); + phys_dsgl = get_space_for_phys_dsgl(nents); kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); @@ -1020,8 +1047,7 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, goto complete; } bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1, - SPACE_LEFT(ablkctx->enckey_len), - &wrparam.snent, &reqctx->dst_nents); + SPACE_LEFT(ablkctx->enckey_len)); if ((bytes + reqctx->processed) >= req->nbytes) bytes = req->nbytes - reqctx->processed; else @@ -1060,8 +1086,6 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, chcr_send_wr(skb); return 0; complete: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; req->base.complete(&req->base, err); return err; } @@ -1077,9 +1101,8 @@ static int process_cipher(struct ablkcipher_request *req, struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); struct ablk_ctx *ablkctx = ABLK_CTX(ctx); struct cipher_wr_param wrparam; - int bytes, nents, err = -EINVAL; + int bytes, err = -EINVAL; - reqctx->newdstsg = NULL; reqctx->processed = 0; if (!req->info) goto error; @@ -1091,18 +1114,9 @@ static int process_cipher(struct ablkcipher_request *req, goto error; } wrparam.srcsg = req->src; - if (is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return PTR_ERR(reqctx->newdstsg); - reqctx->dstsg = reqctx->newdstsg; - } else { reqctx->dstsg = req->dst; - } bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG, - SPACE_LEFT(ablkctx->enckey_len), - &wrparam.snent, - &reqctx->dst_nents); + SPACE_LEFT(ablkctx->enckey_len)); if ((bytes + reqctx->processed) >= req->nbytes) bytes = req->nbytes - reqctx->processed; else @@ -1152,8 +1166,6 @@ static int process_cipher(struct ablkcipher_request *req, return 0; error: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return err; } @@ -1824,63 +1836,6 @@ static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) } } -static int is_newsg(struct scatterlist *sgl, unsigned int *newents) -{ - int nents = 0; - int ret = 0; - - while (sgl) { - if (sgl->length > CHCR_SG_SIZE) - ret = 1; - nents += DIV_ROUND_UP(sgl->length, CHCR_SG_SIZE); - sgl = sg_next(sgl); - } - *newents = nents; - return ret; -} - -static inline void free_new_sg(struct scatterlist *sgl) -{ - kfree(sgl); -} - -static struct scatterlist *alloc_new_sg(struct scatterlist *sgl, - unsigned int nents) -{ - struct scatterlist *newsg, *sg; - int i, len, processed = 0; - struct page *spage; - int offset; - - newsg = kmalloc_array(nents, sizeof(struct scatterlist), GFP_KERNEL); - if (!newsg) - return ERR_PTR(-ENOMEM); - sg = newsg; - sg_init_table(sg, nents); - offset = sgl->offset; - spage = sg_page(sgl); - for (i = 0; i < nents; i++) { - len = min_t(u32, sgl->length - processed, CHCR_SG_SIZE); - sg_set_page(sg, spage, len, offset); - processed += len; - offset += len; - if (offset >= PAGE_SIZE) { - offset = offset % PAGE_SIZE; - spage++; - } - if (processed == sgl->length) { - processed = 0; - sgl = sg_next(sgl); - if (!sgl) - break; - spage = sg_page(sgl); - offset = sgl->offset; - } - sg = sg_next(sg); - } - return newsg; -} - static int chcr_copy_assoc(struct aead_request *req, struct chcr_aead_ctx *ctx) { @@ -1953,7 +1908,6 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, GFP_ATOMIC; struct adapter *adap = padap(ctx->dev); - reqctx->newdstsg = NULL; dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : authsize); if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0)) @@ -1965,24 +1919,13 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, if (src_nent < 0) goto err; src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); - + reqctx->dst = src; if (req->src != req->dst) { error = chcr_copy_assoc(req, aeadctx); if (error) return ERR_PTR(error); - } - if (dst_size && is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return ERR_CAST(reqctx->newdstsg); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - reqctx->newdstsg, req->assoclen); - } else { - if (req->src == req->dst) - reqctx->dst = src; - else - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - req->dst, req->assoclen); + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, + req->assoclen); } if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) { null = 1; @@ -1995,7 +1938,9 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, error = -EINVAL; goto err; } - dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); + nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + + (op_type ? -authsize : authsize)) : 0; + dst_size = get_space_for_phys_dsgl(nents); kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) - sizeof(chcr_req->key_ctx); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); @@ -2004,8 +1949,6 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8), op_type)) { atomic_inc(&adap->chcr_stats.fallback); - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(chcr_aead_fallback(req, op_type)); } skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); @@ -2088,8 +2031,6 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, /* ivmap_fail: */ kfree_skb(skb); err: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(error); } @@ -2307,7 +2248,6 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : authsize); - reqctx->newdstsg = NULL; if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) goto err; src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); @@ -2316,25 +2256,15 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, sub_type = get_aead_subtype(tfm); src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); + reqctx->dst = src; if (req->src != req->dst) { error = chcr_copy_assoc(req, aeadctx); if (error) { pr_err("AAD copy to destination buffer fails\n"); return ERR_PTR(error); } - } - if (dst_size && is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return ERR_CAST(reqctx->newdstsg); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - reqctx->newdstsg, req->assoclen); - } else { - if (req->src == req->dst) - reqctx->dst = src; - else - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - req->dst, req->assoclen); + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, + req->assoclen); } reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + (op_type ? -authsize : authsize)); @@ -2346,8 +2276,9 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type); if (error) goto err; - - dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); + nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + + (op_type ? -authsize : authsize)) : 0; + dst_size = get_space_for_phys_dsgl(nents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG, @@ -2355,8 +2286,6 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8), op_type)) { atomic_inc(&adap->chcr_stats.fallback); - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(chcr_aead_fallback(req, op_type)); } @@ -2402,8 +2331,6 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, dstmap_fail: kfree_skb(skb); err: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(error); } @@ -2432,7 +2359,6 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, GFP_ATOMIC; struct adapter *adap = padap(ctx->dev); - reqctx->newdstsg = NULL; dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize); /* validate key size */ @@ -2446,26 +2372,14 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, goto err; src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen); + reqctx->dst = src; if (req->src != req->dst) { error = chcr_copy_assoc(req, aeadctx); if (error) return ERR_PTR(error); + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, + req->assoclen); } - - if (dst_size && is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return ERR_CAST(reqctx->newdstsg); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - reqctx->newdstsg, assoclen); - } else { - if (req->src == req->dst) - reqctx->dst = src; - else - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - req->dst, assoclen); - } - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + (op_type ? -authsize : authsize)); if (reqctx->dst_nents < 0) { @@ -2474,8 +2388,9 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, goto err; } - - dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); + nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + + (op_type ? -authsize : authsize)) : 0; + dst_size = get_space_for_phys_dsgl(nents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) + AEAD_H_SIZE; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); @@ -2484,8 +2399,6 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8), op_type)) { atomic_inc(&adap->chcr_stats.fallback); - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(chcr_aead_fallback(req, op_type)); } skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); @@ -2563,8 +2476,6 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, /* ivmap_fail: */ kfree_skb(skb); err: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(error); } diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index 54851ec..d1a787b 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -221,7 +221,7 @@ #define MAX_WR_SIZE 512 #define ROUND_16(bytes) ((bytes) & 0xFFFFFFF0) #define MAX_DSGL_ENT 32 -#define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 2) +#define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 1) #define MIN_CIPHER_SG 1 /* IV */ #define MIN_AUTH_SG 2 /*IV + AAD*/ #define MIN_GCM_SG 2 /* IV + AAD*/ @@ -261,7 +261,6 @@ struct cipher_wr_param { struct scatterlist *srcsg; char *iv; int bytes; - short int snent; unsigned short qid; }; enum { diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h index c9a19b2..94e7412 100644 --- a/drivers/crypto/chelsio/chcr_core.h +++ b/drivers/crypto/chelsio/chcr_core.h @@ -89,7 +89,7 @@ struct uld_ctx { struct chcr_dev *dev; }; -struct uld_ctx * assign_chcr_device(void); +struct uld_ctx *assign_chcr_device(void); int chcr_send_wr(struct sk_buff *skb); int start_crypto(void); int stop_crypto(void); diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index b3722b3..8f436f8 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -166,7 +166,6 @@ struct ablk_ctx { struct chcr_aead_reqctx { struct sk_buff *skb; struct scatterlist *dst; - struct scatterlist *newdstsg; struct scatterlist srcffwd[2]; struct scatterlist dstffwd[2]; short int dst_nents; @@ -245,7 +244,6 @@ struct chcr_blkcipher_req_ctx { struct scatterlist dstffwd[2]; struct scatterlist *dstsg; struct scatterlist *dst; - struct scatterlist *newdstsg; unsigned int processed; unsigned int last_req_len; unsigned int op; @@ -291,10 +289,6 @@ static int chcr_aead_op(struct aead_request *req_base, int size, create_wr_t create_wr_fn); static inline int get_aead_subtype(struct crypto_aead *aead); -static int is_newsg(struct scatterlist *sgl, unsigned int *newents); -static struct scatterlist *alloc_new_sg(struct scatterlist *sgl, - unsigned int nents); -static inline void free_new_sg(struct scatterlist *sgl); static int chcr_handle_cipher_resp(struct ablkcipher_request *req, unsigned char *input, int err); #endif /* __CHCR_CRYPTO_H__ */ -- 2.1.4 ^ permalink raw reply related [flat|nested] 18+ messages in thread
* [PATCH 6/7] crypto:chelsio:Move DMA un/mapping to chcr from lld cxgb4 driver 2017-10-03 6:46 [PATCH 0/7]crypto:chelsio: Bugs fixes Harsh Jain ` (4 preceding siblings ...) 2017-10-03 6:46 ` [PATCH 5/7] crypto:chelsio:Remove allocation of sg list to implement 2K limit of dsgl header Harsh Jain @ 2017-10-03 6:46 ` Harsh Jain 2017-10-03 6:46 ` [PATCH 7/7] crypto:chelsio: Fix memory leak Harsh Jain [not found] ` <cover.1507449374.git.harsh@chelsio.com> 7 siblings, 0 replies; 18+ messages in thread From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw) To: herbert, linux-crypto, netdev; +Cc: Harsh Jain, Ganesh Goudar Allow chcr to do DMA mapping/Unmapping instead of lld cxgb4. It moves "Copy AAD to dst buffer" requirement from driver to firmware. Signed-off-by: Ganesh Goudar <ganeshgr@chelsio.com> Signed-off-by: Harsh Jain <harsh@chelsio.com> --- drivers/crypto/chelsio/chcr_algo.c | 1645 ++++++++++++++++++------------ drivers/crypto/chelsio/chcr_algo.h | 44 +- drivers/crypto/chelsio/chcr_crypto.h | 114 ++- drivers/net/ethernet/chelsio/cxgb4/sge.c | 8 +- 4 files changed, 1116 insertions(+), 695 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index b13991d..646dfff 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -70,6 +70,8 @@ #include "chcr_algo.h" #include "chcr_crypto.h" +#define IV AES_BLOCK_SIZE + static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) { return ctx->crypto_ctx->aeadctx; @@ -102,7 +104,7 @@ static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx) static inline int is_ofld_imm(const struct sk_buff *skb) { - return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN); + return (skb->len <= SGE_MAX_WR_LEN); } /* @@ -117,21 +119,92 @@ static inline unsigned int sgl_len(unsigned int n) return (3 * n) / 2 + (n & 1) + 2; } -static int dstsg_2k(struct scatterlist *sgl, unsigned int reqlen) +static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen, + unsigned int entlen, + unsigned int skip) { int nents = 0; unsigned int less; + unsigned int skip_len = 0; - while (sgl && reqlen) { - less = min(reqlen, sgl->length); - nents += DIV_ROUND_UP(less, CHCR_SG_SIZE); - reqlen -= less; - sgl = sg_next(sgl); + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; + sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; + } } + while (sg && reqlen) { + less = min(reqlen, sg_dma_len(sg) - skip_len); + nents += DIV_ROUND_UP(less, entlen); + reqlen -= less; + skip_len = 0; + sg = sg_next(sg); + } return nents; } +static inline void chcr_handle_ahash_resp(struct ahash_request *req, + unsigned char *input, + int err) +{ + struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); + int digestsize, updated_digestsize; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); + + if (input == NULL) + goto out; + reqctx = ahash_request_ctx(req); + digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); + if (reqctx->is_sg_map) + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + if (reqctx->dma_addr) + dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->dma_addr, + reqctx->dma_len, DMA_TO_DEVICE); + reqctx->dma_addr = 0; + updated_digestsize = digestsize; + if (digestsize == SHA224_DIGEST_SIZE) + updated_digestsize = SHA256_DIGEST_SIZE; + else if (digestsize == SHA384_DIGEST_SIZE) + updated_digestsize = SHA512_DIGEST_SIZE; + if (reqctx->result == 1) { + reqctx->result = 0; + memcpy(req->result, input + sizeof(struct cpl_fw6_pld), + digestsize); + } else { + memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld), + updated_digestsize); + } +out: + req->base.complete(&req->base, err); + + } + +static inline void chcr_handle_aead_resp(struct aead_request *req, + unsigned char *input, + int err) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm)); + + + chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op); + if (reqctx->b0_dma) + dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma, + reqctx->b0_len, DMA_BIDIRECTIONAL); + if (reqctx->verify == VERIFY_SW) { + chcr_verify_tag(req, input, &err); + reqctx->verify = VERIFY_HW; +} + req->base.complete(&req->base, err); + +} static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) { u8 temp[SHA512_DIGEST_SIZE]; @@ -166,27 +239,11 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, { struct crypto_tfm *tfm = req->tfm; struct chcr_context *ctx = crypto_tfm_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_req_ctx ctx_req; - unsigned int digestsize, updated_digestsize; struct adapter *adap = padap(ctx->dev); switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_AEAD: - ctx_req.req.aead_req = aead_request_cast(req); - ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req); - dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst, - ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE); - if (ctx_req.ctx.reqctx->skb) { - kfree_skb(ctx_req.ctx.reqctx->skb); - ctx_req.ctx.reqctx->skb = NULL; - } - if (ctx_req.ctx.reqctx->verify == VERIFY_SW) { - chcr_verify_tag(ctx_req.req.aead_req, input, - &err); - ctx_req.ctx.reqctx->verify = VERIFY_HW; - } - ctx_req.req.aead_req->base.complete(req, err); + chcr_handle_aead_resp(aead_request_cast(req), input, err); break; case CRYPTO_ALG_TYPE_ABLKCIPHER: @@ -195,60 +252,13 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, break; case CRYPTO_ALG_TYPE_AHASH: - ctx_req.req.ahash_req = ahash_request_cast(req); - ctx_req.ctx.ahash_ctx = - ahash_request_ctx(ctx_req.req.ahash_req); - digestsize = - crypto_ahash_digestsize(crypto_ahash_reqtfm( - ctx_req.req.ahash_req)); - updated_digestsize = digestsize; - if (digestsize == SHA224_DIGEST_SIZE) - updated_digestsize = SHA256_DIGEST_SIZE; - else if (digestsize == SHA384_DIGEST_SIZE) - updated_digestsize = SHA512_DIGEST_SIZE; - if (ctx_req.ctx.ahash_ctx->skb) { - kfree_skb(ctx_req.ctx.ahash_ctx->skb); - ctx_req.ctx.ahash_ctx->skb = NULL; - } - if (ctx_req.ctx.ahash_ctx->result == 1) { - ctx_req.ctx.ahash_ctx->result = 0; - memcpy(ctx_req.req.ahash_req->result, input + - sizeof(struct cpl_fw6_pld), - digestsize); - } else { - memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input + - sizeof(struct cpl_fw6_pld), - updated_digestsize); + chcr_handle_ahash_resp(ahash_request_cast(req), input, err); } - ctx_req.req.ahash_req->base.complete(req, err); - break; - } atomic_inc(&adap->chcr_stats.complete); return err; } -/* - * calc_tx_flits_ofld - calculate # of flits for an offload packet - * @skb: the packet - * Returns the number of flits needed for the given offload packet. - * These packets are already fully constructed and no additional headers - * will be added. - */ -static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) -{ - unsigned int flits, cnt; - - if (is_ofld_imm(skb)) - return DIV_ROUND_UP(skb->len, 8); - - flits = skb_transport_offset(skb) / 8; /* headers */ - cnt = skb_shinfo(skb)->nr_frags; - if (skb_tail_pointer(skb) != skb_transport_header(skb)) - cnt++; - return flits + sgl_len(cnt); -} - -static inline void get_aes_decrypt_key(unsigned char *dec_key, +static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength) { @@ -395,64 +405,193 @@ static inline int is_hmac(struct crypto_tfm *tfm) return 0; } -static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl, - struct scatterlist *sg, - struct phys_sge_parm *sg_param) +static inline void dsgl_walk_init(struct dsgl_walk *walk, + struct cpl_rx_phys_dsgl *dsgl) { - struct phys_sge_pairs *to; - unsigned int len = 0, left_size = sg_param->obsize; - unsigned int j = 0; - int offset, ent_len; + walk->dsgl = dsgl; + walk->nents = 0; + walk->to = (struct phys_sge_pairs *)(dsgl + 1); +} + +static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid) +{ + struct cpl_rx_phys_dsgl *phys_cpl; + + phys_cpl = walk->dsgl; phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); - to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl + - sizeof(struct cpl_rx_phys_dsgl)); + phys_cpl->pcirlxorder_to_noofsgentr = + htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | + CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | + CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | + CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | + CPL_RX_PHYS_DSGL_DCAID_V(0) | + CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents)); + phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; + phys_cpl->rss_hdr_int.qid = htons(qid); + phys_cpl->rss_hdr_int.hash_val = 0; +} + +static inline void dsgl_walk_add_page(struct dsgl_walk *walk, + size_t size, + dma_addr_t *addr) +{ + int j; + + if (!size) + return; + j = walk->nents; + walk->to->len[j % 8] = htons(size); + walk->to->addr[j % 8] = cpu_to_be64(*addr); + j++; + if ((j % 8) == 0) + walk->to++; + walk->nents = j; +} + +static void dsgl_walk_add_sg(struct dsgl_walk *walk, + struct scatterlist *sg, + unsigned int slen, + unsigned int skip) +{ + int skip_len = 0; + unsigned int left_size = slen, len = 0; + unsigned int j = walk->nents; + int offset, ent_len; + + if (!slen) + return; + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; + sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; + } + } + while (left_size && sg) { - len = min_t(u32, left_size, sg_dma_len(sg)); + len = min_t(u32, left_size, sg_dma_len(sg) - skip_len); offset = 0; while (len) { - ent_len = min_t(u32, len, CHCR_SG_SIZE); - to->len[j % 8] = htons(ent_len); - to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + - offset); + ent_len = min_t(u32, len, CHCR_DST_SG_SIZE); + walk->to->len[j % 8] = htons(ent_len); + walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + + offset + skip_len); offset += ent_len; len -= ent_len; j++; if ((j % 8) == 0) - to++; + walk->to++; } - left_size -= min(left_size, sg_dma_len(sg)); + walk->last_sg = sg; + walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) - + skip_len) + skip_len; + left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len); + skip_len = 0; sg = sg_next(sg); } - phys_cpl->pcirlxorder_to_noofsgentr = - htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | - CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | - CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | - CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | - CPL_RX_PHYS_DSGL_DCAID_V(0) | - CPL_RX_PHYS_DSGL_NOOFSGENTR_V(j)); - phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; - phys_cpl->rss_hdr_int.qid = htons(sg_param->qid); - phys_cpl->rss_hdr_int.hash_val = 0; + walk->nents = j; +} + +static inline void ulptx_walk_init(struct ulptx_walk *walk, + struct ulptx_sgl *ulp) +{ + walk->sgl = ulp; + walk->nents = 0; + walk->pair_idx = 0; + walk->pair = ulp->sge; + walk->last_sg = NULL; + walk->last_sg_len = 0; +} + +static inline void ulptx_walk_end(struct ulptx_walk *walk) +{ + walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | + ULPTX_NSGE_V(walk->nents)); +} + +static inline void ulptx_walk_add_page(struct ulptx_walk *walk, + size_t size, + dma_addr_t *addr) +{ + if (!size) + return; + + if (walk->nents == 0) { + walk->sgl->len0 = cpu_to_be32(size); + walk->sgl->addr0 = cpu_to_be64(*addr); + } else { + walk->pair->addr[walk->pair_idx] = cpu_to_be64(*addr); + walk->pair->len[walk->pair_idx] = cpu_to_be32(size); + walk->pair_idx = !walk->pair_idx; + if (!walk->pair_idx) + walk->pair++; + } + walk->nents++; } -static inline int map_writesg_phys_cpl(struct device *dev, - struct cpl_rx_phys_dsgl *phys_cpl, +static void ulptx_walk_add_sg(struct ulptx_walk *walk, struct scatterlist *sg, - struct phys_sge_parm *sg_param) + unsigned int len, + unsigned int skip) { - if (!sg || !sg_param->nents) - return -EINVAL; + int small; + int skip_len = 0; + unsigned int sgmin; - sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE); - if (sg_param->nents == 0) { - pr_err("CHCR : DMA mapping failed\n"); - return -EINVAL; + if (!len) + return; + + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; + sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; + } + } + if (walk->nents == 0) { + small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len); + sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); + walk->sgl->len0 = cpu_to_be32(sgmin); + walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len); + walk->nents++; + len -= sgmin; + walk->last_sg = sg; + walk->last_sg_len = sgmin + skip_len; + skip_len += sgmin; + if (sg_dma_len(sg) == skip_len) { + sg = sg_next(sg); + skip_len = 0; + } + } + + while (sg && len) { + small = min(sg_dma_len(sg) - skip_len, len); + sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); + walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin); + walk->pair->addr[walk->pair_idx] = + cpu_to_be64(sg_dma_address(sg) + skip_len); + walk->pair_idx = !walk->pair_idx; + walk->nents++; + if (!walk->pair_idx) + walk->pair++; + len -= sgmin; + skip_len += sgmin; + walk->last_sg = sg; + walk->last_sg_len = skip_len; + if (sg_dma_len(sg) == skip_len) { + sg = sg_next(sg); + skip_len = 0; + } } - write_phys_cpl(phys_cpl, sg, sg_param); - return 0; } static inline int get_aead_subtype(struct crypto_aead *aead) @@ -472,45 +611,6 @@ static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm) return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; } -static inline void write_buffer_to_skb(struct sk_buff *skb, - unsigned int *frags, - char *bfr, - u8 bfr_len) -{ - skb->len += bfr_len; - skb->data_len += bfr_len; - skb->truesize += bfr_len; - get_page(virt_to_page(bfr)); - skb_fill_page_desc(skb, *frags, virt_to_page(bfr), - offset_in_page(bfr), bfr_len); - (*frags)++; -} - - -static inline void -write_sg_to_skb(struct sk_buff *skb, unsigned int *frags, - struct scatterlist *sg, unsigned int count) -{ - struct page *spage; - unsigned int page_len; - - skb->len += count; - skb->data_len += count; - skb->truesize += count; - - while (count > 0) { - if (!sg || (!(sg->length))) - break; - spage = sg_page(sg); - get_page(spage); - page_len = min(sg->length, count); - skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len); - (*frags)++; - count -= page_len; - sg = sg_next(sg); - } -} - static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) { struct adapter *adap = netdev2adap(dev); @@ -546,32 +646,46 @@ static int generate_copy_rrkey(struct ablk_ctx *ablkctx, static int chcr_sg_ent_in_wr(struct scatterlist *src, struct scatterlist *dst, unsigned int minsg, - unsigned int space) + unsigned int space, + unsigned int srcskip, + unsigned int dstskip) { int srclen = 0, dstlen = 0; - int srcsg = minsg, dstsg = 0; + int srcsg = minsg, dstsg = minsg; int offset = 0, less; - while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) && + if (sg_dma_len(src) == srcskip) { + src = sg_next(src); + srcskip = 0; + } + + if (sg_dma_len(dst) == dstskip) { + dst = sg_next(dst); + dstskip = 0; + } + + while (src && dst && space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { - srclen += src->length; + srclen += (sg_dma_len(src) - srcskip); srcsg++; offset = 0; while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { if (srclen <= dstlen) break; - less = min_t(unsigned int, dst->length - offset, - CHCR_SG_SIZE); + less = min_t(unsigned int, sg_dma_len(dst) - offset - + dstskip, CHCR_DST_SG_SIZE); dstlen += less; offset += less; - if (offset == dst->length) { + if (offset == sg_dma_len(dst)) { dst = sg_next(dst); offset = 0; } dstsg++; + dstskip = 0; } src = sg_next(src); + srcskip = 0; } return min(srclen, dstlen); } @@ -601,24 +715,22 @@ static int chcr_cipher_fallback(struct crypto_skcipher *cipher, } static inline void create_wreq(struct chcr_context *ctx, struct chcr_wr *chcr_req, - void *req, struct sk_buff *skb, + struct crypto_async_request *req, + unsigned int imm, int hash_sz, + unsigned int len16, unsigned int sc_len, unsigned int lcb) { struct uld_ctx *u_ctx = ULD_CTX(ctx); int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; - unsigned int immdatalen = 0; - if (is_ofld_imm(skb)) - immdatalen = skb->data_len; chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE; chcr_req->wreq.pld_size_hash_size = htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); chcr_req->wreq.len16_pkd = - htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP( - (calc_tx_flits_ofld(skb) * 8), 16))); + htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16))); chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid, @@ -626,13 +738,12 @@ static inline void create_wreq(struct chcr_context *ctx, chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id, qid); - chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8), - 16) - ((sizeof(chcr_req->wreq)) >> 4))); + chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) - + ((sizeof(chcr_req->wreq)) >> 4))); - chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen); + chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm); chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + - sizeof(chcr_req->key_ctx) + - sc_len + immdatalen); + sizeof(chcr_req->key_ctx) + sc_len); } /** @@ -645,49 +756,52 @@ static inline void create_wreq(struct chcr_context *ctx, static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; + struct ulptx_sgl *ulptx; struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(wrparam->req); - struct phys_sge_parm sg_param; - unsigned int frags = 0, transhdr_len, phys_dsgl; + unsigned int temp = 0, transhdr_len, dst_size; int error; int nents; - unsigned int ivsize = AES_BLOCK_SIZE, kctx_len; + unsigned int kctx_len; gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(c_ctx(tfm)->dev); - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, wrparam->bytes); - nents = dstsg_2k(reqctx->dst, wrparam->bytes); - phys_dsgl = get_space_for_phys_dsgl(nents); + nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE, + reqctx->dst_ofst); + dst_size = get_space_for_phys_dsgl(nents + 1); kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); - transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); + nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes, + CHCR_SRC_SG_SIZE, reqctx->src_ofst); + temp = reqctx->imm ? (DIV_ROUND_UP((IV + wrparam->req->nbytes), 16) + * 16) : (sgl_len(nents + MIN_CIPHER_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); chcr_req = __skb_put_zero(skb, transhdr_len); chcr_req->sec_cpl.op_ivinsrtofst = - FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 1); + FILL_SEC_CPL_OP_IVINSR(c_ctx(tfm)->dev->rx_channel_id, 2, 1); - chcr_req->sec_cpl.pldlen = htonl(ivsize + wrparam->bytes); + chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes); chcr_req->sec_cpl.aadstart_cipherstop_hi = - FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0); + FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0); chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0, ablkctx->ciph_mode, - 0, 0, ivsize >> 1); + 0, 0, IV >> 1); chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0, - 0, 1, phys_dsgl); + 0, 0, dst_size); chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr; if ((reqctx->op == CHCR_DECRYPT_OP) && @@ -712,26 +826,18 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) } } phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = wrparam->bytes; - sg_param.qid = wrparam->qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) - goto map_fail1; + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); + chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam); + chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid); - skb_set_transport_header(skb, transhdr_len); - write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); - write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes); atomic_inc(&adap->chcr_stats.cipher_rqst); - create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, 0, - sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl + kctx_len, + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + +(reqctx->imm ? (IV + wrparam->bytes) : 0); + create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0, + transhdr_len, temp, ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); reqctx->skb = skb; - skb_get(skb); return skb; -map_fail1: - kfree_skb(skb); err: return ERR_PTR(error); } @@ -756,8 +862,7 @@ static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher, unsigned int keylen) { struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); int err = 0; crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK); @@ -775,8 +880,7 @@ static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned int ck_size, context_size; u16 alignment = 0; int err; @@ -808,8 +912,7 @@ static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned int ck_size, context_size; u16 alignment = 0; int err; @@ -840,8 +943,7 @@ static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned int ck_size, context_size; u16 alignment = 0; int err; @@ -908,8 +1010,7 @@ static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes) static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); struct crypto_cipher *cipher; int ret, i; @@ -926,8 +1027,7 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) ret = crypto_cipher_setkey(cipher, key, keylen); if (ret) goto out; - - crypto_cipher_encrypt_one(cipher, iv, iv); + /*H/W sends the encrypted IV in dsgl when AADIVDROP bit is 0*/ for (i = 0; i < round8; i++) gf128mul_x8_ble((le128 *)iv, (le128 *)iv); @@ -1005,64 +1105,60 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, unsigned char *input, int err) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct sk_buff *skb; struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input; struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); struct cipher_wr_param wrparam; int bytes; - dma_unmap_sg(&u_ctx->lldi.pdev->dev, reqctx->dst, reqctx->dst_nents, - DMA_FROM_DEVICE); - - if (reqctx->skb) { - kfree_skb(reqctx->skb); - reqctx->skb = NULL; - } if (err) - goto complete; - + goto unmap; if (req->nbytes == reqctx->processed) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); err = chcr_final_cipher_iv(req, fw6_pld, req->info); goto complete; } if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + c_ctx(tfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { err = -EBUSY; - goto complete; + goto unmap; } } - wrparam.srcsg = scatterwalk_ffwd(reqctx->srcffwd, req->src, - reqctx->processed); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, reqctx->dstsg, - reqctx->processed); - if (!wrparam.srcsg || !reqctx->dst) { - pr_err("Input sg list length less that nbytes\n"); - err = -EINVAL; - goto complete; - } - bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1, - SPACE_LEFT(ablkctx->enckey_len)); + if (!reqctx->imm) { + bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 1, + SPACE_LEFT(ablkctx->enckey_len), + reqctx->src_ofst, reqctx->dst_ofst); if ((bytes + reqctx->processed) >= req->nbytes) bytes = req->nbytes - reqctx->processed; else bytes = ROUND_16(bytes); + } else { + /*CTR mode counter overfloa*/ + bytes = req->nbytes - reqctx->processed; + } + dma_sync_single_for_cpu(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv); + dma_sync_single_for_device(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); if (err) - goto complete; + goto unmap; if (unlikely(bytes == 0)) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); err = chcr_cipher_fallback(ablkctx->sw_cipher, req->base.flags, - wrparam.srcsg, - reqctx->dst, - req->nbytes - reqctx->processed, - reqctx->iv, + req->src, + req->dst, + req->nbytes, + req->info, reqctx->op); goto complete; } @@ -1070,21 +1166,23 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == CRYPTO_ALG_SUB_TYPE_CTR) bytes = adjust_ctr_overflow(reqctx->iv, bytes); - reqctx->processed += bytes; - reqctx->last_req_len = bytes; - wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; + wrparam.qid = u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx]; wrparam.req = req; wrparam.bytes = bytes; skb = create_cipher_wr(&wrparam); if (IS_ERR(skb)) { pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); err = PTR_ERR(skb); - goto complete; + goto unmap; } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); chcr_send_wr(skb); + reqctx->last_req_len = bytes; + reqctx->processed += bytes; return 0; +unmap: + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); complete: req->base.complete(&req->base, err); return err; @@ -1098,8 +1196,7 @@ static int process_cipher(struct ablkcipher_request *req, struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct cipher_wr_param wrparam; int bytes, err = -EINVAL; @@ -1113,16 +1210,41 @@ static int process_cipher(struct ablkcipher_request *req, ablkctx->enckey_len, req->nbytes, ivsize); goto error; } - wrparam.srcsg = req->src; - reqctx->dstsg = req->dst; - bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG, - SPACE_LEFT(ablkctx->enckey_len)); + chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); + if (req->nbytes < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) + + AES_MIN_KEY_SIZE + + sizeof(struct cpl_rx_phys_dsgl) + + /*Min dsgl size*/ + 32))) { + /* Can be sent as Imm*/ + unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len; + + dnents = sg_nents_xlen(req->dst, req->nbytes, + CHCR_DST_SG_SIZE, 0); + dnents += 1; // IV + phys_dsgl = get_space_for_phys_dsgl(dnents); + kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); + transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); + reqctx->imm = (transhdr_len + IV + req->nbytes) <= + SGE_MAX_WR_LEN; + bytes = IV + req->nbytes; + + } else { + reqctx->imm = 0; + } + + if (!reqctx->imm) { + bytes = chcr_sg_ent_in_wr(req->src, req->dst, + MIN_CIPHER_SG, + SPACE_LEFT(ablkctx->enckey_len), + 0, 0); if ((bytes + reqctx->processed) >= req->nbytes) bytes = req->nbytes - reqctx->processed; else bytes = ROUND_16(bytes); - if (unlikely(bytes > req->nbytes)) + } else { bytes = req->nbytes; + } if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == CRYPTO_ALG_SUB_TYPE_CTR) { bytes = adjust_ctr_overflow(req->info, bytes); @@ -1139,9 +1261,11 @@ static int process_cipher(struct ablkcipher_request *req, } else { - memcpy(reqctx->iv, req->info, ivsize); + memcpy(reqctx->iv, req->info, IV); } if (unlikely(bytes == 0)) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); err = chcr_cipher_fallback(ablkctx->sw_cipher, req->base.flags, req->src, @@ -1151,20 +1275,25 @@ static int process_cipher(struct ablkcipher_request *req, op_type); goto error; } - reqctx->processed = bytes; - reqctx->last_req_len = bytes; - reqctx->dst = reqctx->dstsg; reqctx->op = op_type; + reqctx->srcsg = req->src; + reqctx->dstsg = req->dst; + reqctx->src_ofst = 0; + reqctx->dst_ofst = 0; wrparam.qid = qid; wrparam.req = req; wrparam.bytes = bytes; *skb = create_cipher_wr(&wrparam); if (IS_ERR(*skb)) { err = PTR_ERR(*skb); - goto error; + goto unmap; } + reqctx->processed = bytes; + reqctx->last_req_len = bytes; return 0; +unmap: + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); error: return err; } @@ -1172,23 +1301,22 @@ static int process_cipher(struct ablkcipher_request *req, static int chcr_aes_encrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); struct sk_buff *skb = NULL; int err; - struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + c_ctx(tfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } - err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb, - CHCR_ENCRYPT_OP); + err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx], + &skb, CHCR_ENCRYPT_OP); if (err || !skb) return err; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -1196,23 +1324,22 @@ static int chcr_aes_encrypt(struct ablkcipher_request *req) static int chcr_aes_decrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); struct sk_buff *skb = NULL; int err; if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + c_ctx(tfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } - err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb, - CHCR_DECRYPT_OP); + err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx], + &skb, CHCR_DECRYPT_OP); if (err || !skb) return err; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -1360,17 +1487,19 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); + struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); struct sk_buff *skb = NULL; + struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); struct chcr_wr *chcr_req; - unsigned int frags = 0, transhdr_len, iopad_alignment = 0; + struct ulptx_sgl *ulptx; + unsigned int nents = 0, transhdr_len, iopad_alignment = 0; unsigned int digestsize = crypto_ahash_digestsize(tfm); - unsigned int kctx_len = 0; + unsigned int kctx_len = 0, temp = 0; u8 hash_size_in_response = 0; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(h_ctx(tfm)->dev); + int error = 0; iopad_alignment = KEYCTX_ALIGN_PAD(digestsize); kctx_len = param->alg_prm.result_size + iopad_alignment; @@ -1382,15 +1511,22 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, else hash_size_in_response = param->alg_prm.result_size; transhdr_len = HASH_TRANSHDR_SIZE(kctx_len); - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + req_ctx->imm = (transhdr_len + param->bfr_len + param->sg_len) <= + SGE_MAX_WR_LEN; + nents = sg_nents_xlen(req->src, param->sg_len, CHCR_SRC_SG_SIZE, 0); + nents += param->bfr_len ? 1 : 0; + transhdr_len += req_ctx->imm ? (DIV_ROUND_UP((param->bfr_len + + param->sg_len), 16) * 16) : + (sgl_len(nents) * 8); + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) - return skb; - - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); + return ERR_PTR(-ENOMEM); chcr_req = __skb_put_zero(skb, transhdr_len); chcr_req->sec_cpl.op_ivinsrtofst = - FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 0); + FILL_SEC_CPL_OP_IVINSR(h_ctx(tfm)->dev->rx_channel_id, 2, 0); chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len); chcr_req->sec_cpl.aadstart_cipherstop_hi = @@ -1419,37 +1555,52 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, ((kctx_len + sizeof(chcr_req->key_ctx)) >> 4)); chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1); - - skb_set_transport_header(skb, transhdr_len); - if (param->bfr_len != 0) - write_buffer_to_skb(skb, &frags, req_ctx->reqbfr, - param->bfr_len); - if (param->sg_len != 0) - write_sg_to_skb(skb, &frags, req->src, param->sg_len); + ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + kctx_len + + DUMMY_BYTES); + if (param->bfr_len != 0) { + req_ctx->dma_addr = dma_map_single(&u_ctx->lldi.pdev->dev, + req_ctx->reqbfr, param->bfr_len, + DMA_TO_DEVICE); + if (dma_mapping_error(&u_ctx->lldi.pdev->dev, + req_ctx->dma_addr)) { + error = -ENOMEM; + goto err; + } + req_ctx->dma_len = param->bfr_len; + } else { + req_ctx->dma_addr = 0; + } + chcr_add_hash_src_ent(req, ulptx, param); + /* Request upto max wr size */ + temp = kctx_len + DUMMY_BYTES + (req_ctx->imm ? (param->sg_len + + param->bfr_len) : 0); atomic_inc(&adap->chcr_stats.digest_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, hash_size_in_response, - DUMMY_BYTES + kctx_len, 0); + create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->imm, + hash_size_in_response, transhdr_len, + temp, 0); req_ctx->skb = skb; - skb_get(skb); return skb; +err: + kfree_skb(skb); + return ERR_PTR(error); } static int chcr_ahash_update(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct uld_ctx *u_ctx = NULL; struct sk_buff *skb; u8 remainder = 0, bs; unsigned int nbytes = req->nbytes; struct hash_wr_param params; + int error; bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + h_ctx(rtfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -1463,7 +1614,9 @@ static int chcr_ahash_update(struct ahash_request *req) req_ctx->reqlen += nbytes; return 0; } - + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) + return -ENOMEM; params.opad_needed = 0; params.more = 1; params.last = 0; @@ -1474,8 +1627,10 @@ static int chcr_ahash_update(struct ahash_request *req) req_ctx->result = 0; req_ctx->data_len += params.sg_len + params.bfr_len; skb = create_hash_wr(req, ¶ms); - if (IS_ERR(skb)) - return PTR_ERR(skb); + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } if (remainder) { u8 *temp; @@ -1489,10 +1644,13 @@ static int chcr_ahash_update(struct ahash_request *req) } req_ctx->reqlen = remainder; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + return error; } static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1) @@ -1509,13 +1667,12 @@ static int chcr_ahash_final(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct hash_wr_param params; struct sk_buff *skb; struct uld_ctx *u_ctx = NULL; u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (is_hmac(crypto_ahash_tfm(rtfm))) params.opad_needed = 1; else @@ -1542,7 +1699,7 @@ static int chcr_ahash_final(struct ahash_request *req) return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -1551,17 +1708,17 @@ static int chcr_ahash_finup(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct uld_ctx *u_ctx = NULL; struct sk_buff *skb; struct hash_wr_param params; u8 bs; + int error; bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + h_ctx(rtfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -1587,34 +1744,41 @@ static int chcr_ahash_finup(struct ahash_request *req) params.last = 1; params.more = 0; } + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) + return -ENOMEM; skb = create_hash_wr(req, ¶ms); - if (IS_ERR(skb)) - return PTR_ERR(skb); - + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + return error; } static int chcr_ahash_digest(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct uld_ctx *u_ctx = NULL; struct sk_buff *skb; struct hash_wr_param params; u8 bs; + int error; rtfm->init(req); bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + h_ctx(rtfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -1623,6 +1787,9 @@ static int chcr_ahash_digest(struct ahash_request *req) params.opad_needed = 1; else params.opad_needed = 0; + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) + return -ENOMEM; params.last = 0; params.more = 0; @@ -1640,13 +1807,17 @@ static int chcr_ahash_digest(struct ahash_request *req) } skb = create_hash_wr(req, ¶ms); - if (IS_ERR(skb)) - return PTR_ERR(skb); - + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + return error; } static int chcr_ahash_export(struct ahash_request *areq, void *out) @@ -1656,6 +1827,8 @@ static int chcr_ahash_export(struct ahash_request *areq, void *out) state->reqlen = req_ctx->reqlen; state->data_len = req_ctx->data_len; + state->is_sg_map = 0; + state->result = 0; memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen); memcpy(state->partial_hash, req_ctx->partial_hash, CHCR_HASH_MAX_DIGEST_SIZE); @@ -1671,6 +1844,8 @@ static int chcr_ahash_import(struct ahash_request *areq, const void *in) req_ctx->data_len = state->data_len; req_ctx->reqbfr = req_ctx->bfr1; req_ctx->skbfr = req_ctx->bfr2; + req_ctx->is_sg_map = 0; + req_ctx->result = 0; memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128); memcpy(req_ctx->partial_hash, state->partial_hash, CHCR_HASH_MAX_DIGEST_SIZE); @@ -1680,8 +1855,7 @@ static int chcr_ahash_import(struct ahash_request *areq, const void *in) static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); + struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); unsigned int digestsize = crypto_ahash_digestsize(tfm); unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); unsigned int i, err = 0, updated_digestsize; @@ -1734,8 +1908,7 @@ static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int key_len) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned short context_size = 0; int err; @@ -1774,6 +1947,7 @@ static int chcr_sha_init(struct ahash_request *areq) req_ctx->skbfr = req_ctx->bfr2; req_ctx->skb = NULL; req_ctx->result = 0; + req_ctx->is_sg_map = 0; copy_hash_init_values(req_ctx->partial_hash, digestsize); return 0; } @@ -1789,8 +1963,7 @@ static int chcr_hmac_init(struct ahash_request *areq) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); + struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm)); unsigned int digestsize = crypto_ahash_digestsize(rtfm); unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); @@ -1836,29 +2009,48 @@ static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) } } -static int chcr_copy_assoc(struct aead_request *req, - struct chcr_aead_ctx *ctx) +static int chcr_aead_common_init(struct aead_request *req, + unsigned short op_type) { - SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null); - - skcipher_request_set_tfm(skreq, ctx->null); - skcipher_request_set_callback(skreq, aead_request_flags(req), - NULL, NULL); - skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen, - NULL); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + int error = -EINVAL; + unsigned int dst_size; + unsigned int authsize = crypto_aead_authsize(tfm); - return crypto_skcipher_encrypt(skreq); + dst_size = req->assoclen + req->cryptlen + (op_type ? + -authsize : authsize); + /* validate key size */ + if (aeadctx->enckey_len == 0) + goto err; + if (op_type && req->cryptlen < authsize) + goto err; + error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); + if (error) { + error = -ENOMEM; + goto err; + } + reqctx->aad_nents = sg_nents_xlen(req->src, req->assoclen, + CHCR_SRC_SG_SIZE, 0); + reqctx->src_nents = sg_nents_xlen(req->src, req->cryptlen, + CHCR_SRC_SG_SIZE, req->assoclen); + return 0; +err: + return error; } -static int chcr_aead_need_fallback(struct aead_request *req, int src_nent, + +static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents, int aadmax, int wrlen, unsigned short op_type) { unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); if (((req->cryptlen - (op_type ? authsize : 0)) == 0) || + dst_nents > MAX_DSGL_ENT || (req->assoclen > aadmax) || - (src_nent > MAX_SKB_FRAGS) || - (wrlen > MAX_WR_SIZE)) + (wrlen > SGE_MAX_WR_LEN)) return 1; return 0; } @@ -1866,8 +2058,7 @@ static int chcr_aead_need_fallback(struct aead_request *req, int src_nent, static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct aead_request *subreq = aead_request_ctx(req); aead_request_set_tfm(subreq, aeadctx->sw_cipher); @@ -1886,84 +2077,75 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; - struct phys_sge_parm sg_param; - struct scatterlist *src; - unsigned int frags = 0, transhdr_len; - unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0; - unsigned int kctx_len = 0, nents; - unsigned short stop_offset = 0; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len; + unsigned int dst_size = 0, temp; + unsigned int kctx_len = 0, dnents; unsigned int assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); - int error = -EINVAL, src_nent; + int error = -EINVAL; int null = 0; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(a_ctx(tfm)->dev); - dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : - authsize); - if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0)) - goto err; + if (req->cryptlen == 0) + return NULL; - if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) - goto err; - src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); - if (src_nent < 0) - goto err; - src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); - reqctx->dst = src; - if (req->src != req->dst) { - error = chcr_copy_assoc(req, aeadctx); - if (error) - return ERR_PTR(error); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, - req->assoclen); - } + reqctx->b0_dma = 0; if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) { null = 1; assoclen = 0; } - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)); - if (reqctx->dst_nents < 0) { - pr_err("AUTHENC:Invalid Destination sg entries\n"); - error = -EINVAL; - goto err; + dst_size = assoclen + req->cryptlen + (op_type ? -authsize : + authsize); + error = chcr_aead_common_init(req, op_type); + if (error) + return ERR_PTR(error); + if (dst_size) { + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, req->cryptlen + + (op_type ? -authsize : authsize), CHCR_DST_SG_SIZE, + req->assoclen); + dnents += MIN_AUTH_SG; // For IV + } else { + dnents = 0; } - nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)) : 0; - dst_size = get_space_for_phys_dsgl(nents); + + dst_size = get_space_for_phys_dsgl(dnents); kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) - sizeof(chcr_req->key_ctx); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); - if (chcr_aead_need_fallback(req, src_nent + MIN_AUTH_SG, - T6_MAX_AAD_SIZE, - transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8), - op_type)) { + reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) < + SGE_MAX_WR_LEN; + temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen), 16) + * 16) : (sgl_len(reqctx->src_nents + reqctx->aad_nents + + MIN_GCM_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, + transhdr_len, op_type)) { atomic_inc(&adap->chcr_stats.fallback); + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); return ERR_PTR(chcr_aead_fallback(req, op_type)); } - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - /* LLD is going to write the sge hdr. */ - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); - - /* Write WR */ chcr_req = __skb_put_zero(skb, transhdr_len); - stop_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; + temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; /* * Input order is AAD,IV and Payload. where IV should be included as @@ -1971,24 +2153,24 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, * to the hardware spec */ chcr_req->sec_cpl.op_ivinsrtofst = - FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, - (ivsize ? (assoclen + 1) : 0)); - chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen); + FILL_SEC_CPL_OP_IVINSR(a_ctx(tfm)->dev->rx_channel_id, 2, + assoclen + 1); + chcr_req->sec_cpl.pldlen = htonl(assoclen + IV + req->cryptlen); chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( assoclen ? 1 : 0, assoclen, - assoclen + ivsize + 1, - (stop_offset & 0x1F0) >> 4); + assoclen + IV + 1, + (temp & 0x1F0) >> 4); chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT( - stop_offset & 0xF, - null ? 0 : assoclen + ivsize + 1, - stop_offset, stop_offset); + temp & 0xF, + null ? 0 : assoclen + IV + 1, + temp, temp); chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, CHCR_SCMD_CIPHER_MODE_AES_CBC, actx->auth_mode, aeadctx->hmac_ctrl, - ivsize >> 1); + IV >> 1); chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, - 0, 1, dst_size); + 0, 0, dst_size); chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; if (op_type == CHCR_ENCRYPT_OP) @@ -2001,39 +2183,312 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4), actx->h_iopad, kctx_len - (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4)); - + memcpy(reqctx->iv, req->iv, IV); phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); - sg_param.qid = qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) - goto dstmap_fail; - - skb_set_transport_header(skb, transhdr_len); - - if (assoclen) { - /* AAD buffer in */ - write_sg_to_skb(skb, &frags, req->src, assoclen); - - } - write_buffer_to_skb(skb, &frags, req->iv, ivsize); - write_sg_to_skb(skb, &frags, src, req->cryptlen); + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); + chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); + chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); atomic_inc(&adap->chcr_stats.cipher_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, size, - sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + + kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, + transhdr_len, temp, 0); reqctx->skb = skb; - skb_get(skb); + reqctx->op = op_type; return skb; -dstmap_fail: - /* ivmap_fail: */ - kfree_skb(skb); err: + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); + return ERR_PTR(error); } +static int chcr_aead_dma_map(struct device *dev, + struct aead_request *req, + unsigned short op_type) +{ + int error; + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(tfm); + int dst_size; + + dst_size = req->assoclen + req->cryptlen + (op_type ? + -authsize : authsize); + if (!req->cryptlen || !dst_size) + return 0; + reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, reqctx->iv_dma)) + return -ENOMEM; + + if (req->src == req->dst) { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + if (!error) + goto err; + } else { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + if (!error) + goto err; + error = dma_map_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + if (!error) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + goto err; + } + } + + return 0; +err: + dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); + return -ENOMEM; +} + +static void chcr_aead_dma_unmap(struct device *dev, + struct aead_request *req, + unsigned short op_type) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(tfm); + int dst_size; + + dst_size = req->assoclen + req->cryptlen + (op_type ? + -authsize : authsize); + if (!req->cryptlen || !dst_size) + return; + + dma_unmap_single(dev, reqctx->iv_dma, IV, + DMA_BIDIRECTIONAL); + if (req->src == req->dst) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + } +} + +static inline void chcr_add_aead_src_ent(struct aead_request *req, + struct ulptx_sgl *ulptx, + unsigned int assoclen, + unsigned short op_type) +{ + struct ulptx_walk ulp_walk; + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + + if (reqctx->imm) { + u8 *buf = (u8 *)ulptx; + + if (reqctx->b0_dma) { + memcpy(buf, reqctx->scratch_pad, reqctx->b0_len); + buf += reqctx->b0_len; + } + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, assoclen, 0); + buf += assoclen; + memcpy(buf, reqctx->iv, IV); + buf += IV; + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, req->cryptlen, req->assoclen); + } else { + ulptx_walk_init(&ulp_walk, ulptx); + if (reqctx->b0_dma) + ulptx_walk_add_page(&ulp_walk, reqctx->b0_len, + &reqctx->b0_dma); + ulptx_walk_add_sg(&ulp_walk, req->src, assoclen, 0); + ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma); + ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen, + req->assoclen); + ulptx_walk_end(&ulp_walk); + } +} + +static inline void chcr_add_aead_dst_ent(struct aead_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + unsigned int assoclen, + unsigned short op_type, + unsigned short qid) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct dsgl_walk dsgl_walk; + unsigned int authsize = crypto_aead_authsize(tfm); + u32 temp; + + dsgl_walk_init(&dsgl_walk, phys_cpl); + if (reqctx->b0_dma) + dsgl_walk_add_page(&dsgl_walk, reqctx->b0_len, &reqctx->b0_dma); + dsgl_walk_add_sg(&dsgl_walk, req->dst, assoclen, 0); + dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma); + temp = req->cryptlen + (op_type ? -authsize : authsize); + dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, req->assoclen); + dsgl_walk_end(&dsgl_walk, qid); +} + +static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, + struct ulptx_sgl *ulptx, + struct cipher_wr_param *wrparam) +{ + struct ulptx_walk ulp_walk; + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + + if (reqctx->imm) { + u8 *buf = (u8 *)ulptx; + + memcpy(buf, reqctx->iv, IV); + buf += IV; + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, wrparam->bytes, reqctx->processed); + } else { + ulptx_walk_init(&ulp_walk, ulptx); + ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma); + ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes, + reqctx->src_ofst); + reqctx->srcsg = ulp_walk.last_sg; + reqctx->src_ofst = ulp_walk.last_sg_len; + ulptx_walk_end(&ulp_walk); + } +} + +static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + struct cipher_wr_param *wrparam, + unsigned short qid) +{ + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + struct dsgl_walk dsgl_walk; + + dsgl_walk_init(&dsgl_walk, phys_cpl); + dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma); + dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes, + reqctx->dst_ofst); + reqctx->dstsg = dsgl_walk.last_sg; + reqctx->dst_ofst = dsgl_walk.last_sg_len; + + dsgl_walk_end(&dsgl_walk, qid); +} + +static inline void chcr_add_hash_src_ent(struct ahash_request *req, + struct ulptx_sgl *ulptx, + struct hash_wr_param *param) +{ + struct ulptx_walk ulp_walk; + struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); + + if (reqctx->imm) { + u8 *buf = (u8 *)ulptx; + + if (param->bfr_len) { + memcpy(buf, reqctx->reqbfr, param->bfr_len); + buf += param->bfr_len; + } + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, param->sg_len, 0); + } else { + ulptx_walk_init(&ulp_walk, ulptx); + if (param->bfr_len) + ulptx_walk_add_page(&ulp_walk, param->bfr_len, + &reqctx->dma_addr); + ulptx_walk_add_sg(&ulp_walk, req->src, param->sg_len, + 0); +// reqctx->srcsg = ulp_walk.last_sg; +// reqctx->src_ofst = ulp_walk.last_sg_len; + ulptx_walk_end(&ulp_walk); + } +} + + +static inline int chcr_hash_dma_map(struct device *dev, + struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + int error = 0; + + if (!req->nbytes) + return 0; + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + if (!error) + return error; + req_ctx->is_sg_map = 1; + return 0; +} + +static inline void chcr_hash_dma_unmap(struct device *dev, + struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + + if (!req->nbytes) + return; + + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + req_ctx->is_sg_map = 0; + +} + + +static int chcr_cipher_dma_map(struct device *dev, + struct ablkcipher_request *req) +{ + int error; + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + + reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, reqctx->iv_dma)) + return -ENOMEM; + + if (req->src == req->dst) { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + if (!error) + goto err; + } else { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + if (!error) + goto err; + error = dma_map_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + if (!error) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + goto err; + } + } + + return 0; +err: + dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); + return -ENOMEM; +} +static void chcr_cipher_dma_unmap(struct device *dev, + struct ablkcipher_request *req) +{ + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + + dma_unmap_single(dev, reqctx->iv_dma, IV, + DMA_BIDIRECTIONAL); + if (req->src == req->dst) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + } +} + static int set_msg_len(u8 *block, unsigned int msglen, int csize) { __be32 data; @@ -2118,15 +2573,13 @@ static int ccm_format_packet(struct aead_request *req, static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, unsigned int dst_size, struct aead_request *req, - unsigned short op_type, - struct chcr_context *chcrctx) + unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); - unsigned int ivsize = AES_BLOCK_SIZE; + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; - unsigned int c_id = chcrctx->dev->rx_channel_id; + unsigned int c_id = a_ctx(tfm)->dev->rx_channel_id; unsigned int ccm_xtra; unsigned char tag_offset = 0, auth_offset = 0; unsigned int assoclen; @@ -2139,7 +2592,7 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, ((assoclen) ? CCM_AAD_FIELD_SIZE : 0); auth_offset = req->cryptlen ? - (assoclen + ivsize + 1 + ccm_xtra) : 0; + (assoclen + IV + 1 + ccm_xtra) : 0; if (op_type == CHCR_DECRYPT_OP) { if (crypto_aead_authsize(tfm) != req->cryptlen) tag_offset = crypto_aead_authsize(tfm); @@ -2149,14 +2602,13 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id, - 2, (ivsize ? (assoclen + 1) : 0) + - ccm_xtra); + 2, assoclen + 1 + ccm_xtra); sec_cpl->pldlen = - htonl(assoclen + ivsize + req->cryptlen + ccm_xtra); + htonl(assoclen + IV + req->cryptlen + ccm_xtra); /* For CCM there wil be b0 always. So AAD start will be 1 always */ sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 1, assoclen + ccm_xtra, assoclen - + ivsize + 1 + ccm_xtra, 0); + + IV + 1 + ccm_xtra, 0); sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, auth_offset, tag_offset, @@ -2165,10 +2617,10 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 0 : 1, cipher_mode, mac_mode, - aeadctx->hmac_ctrl, ivsize >> 1); + aeadctx->hmac_ctrl, IV >> 1); sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, - 1, dst_size); + 0, dst_size); } int aead_ccm_validate_input(unsigned short op_type, @@ -2188,119 +2640,83 @@ int aead_ccm_validate_input(unsigned short op_type, return -EINVAL; } } - if (aeadctx->enckey_len == 0) { - pr_err("CCM: Encryption key not set\n"); - return -EINVAL; - } return 0; } -unsigned int fill_aead_req_fields(struct sk_buff *skb, - struct aead_request *req, - struct scatterlist *src, - unsigned int ivsize, - struct chcr_aead_ctx *aeadctx) -{ - unsigned int frags = 0; - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); - /* b0 and aad length(if available) */ - - write_buffer_to_skb(skb, &frags, reqctx->scratch_pad, CCM_B0_SIZE + - (req->assoclen ? CCM_AAD_FIELD_SIZE : 0)); - if (req->assoclen) { - if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) - write_sg_to_skb(skb, &frags, req->src, - req->assoclen - 8); - else - write_sg_to_skb(skb, &frags, req->src, req->assoclen); - } - write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); - if (req->cryptlen) - write_sg_to_skb(skb, &frags, src, req->cryptlen); - - return frags; -} - static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, unsigned short qid, int size, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; - struct phys_sge_parm sg_param; - struct scatterlist *src; - unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE; - unsigned int dst_size = 0, kctx_len, nents; - unsigned int sub_type; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len; + unsigned int dst_size = 0, kctx_len, dnents, temp; + unsigned int sub_type, assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); - int error = -EINVAL, src_nent; + int error = -EINVAL; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(a_ctx(tfm)->dev); - dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : + reqctx->b0_dma = 0; + sub_type = get_aead_subtype(tfm); + if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) + assoclen -= 8; + dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize); - if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) - goto err; - src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); - if (src_nent < 0) - goto err; + error = chcr_aead_common_init(req, op_type); + if (error) + return ERR_PTR(error); - sub_type = get_aead_subtype(tfm); - src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); - reqctx->dst = src; - if (req->src != req->dst) { - error = chcr_copy_assoc(req, aeadctx); - if (error) { - pr_err("AAD copy to destination buffer fails\n"); - return ERR_PTR(error); - } - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, - req->assoclen); - } - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)); - if (reqctx->dst_nents < 0) { - pr_err("CCM:Invalid Destination sg entries\n"); - error = -EINVAL; - goto err; - } + + reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0); error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type); if (error) goto err; - nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)) : 0; - dst_size = get_space_for_phys_dsgl(nents); + if (dst_size) { + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, req->cryptlen + + (op_type ? -authsize : authsize), + CHCR_DST_SG_SIZE, req->assoclen); + dnents += MIN_CCM_SG; // For IV and B0 + } else { + dnents = 0; + } + dst_size = get_space_for_phys_dsgl(dnents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); - if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG, - T6_MAX_AAD_SIZE - 18, - transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8), - op_type)) { + reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen + + reqctx->b0_len) <= SGE_MAX_WR_LEN; + temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen + + reqctx->b0_len), 16) * 16) : + (sgl_len(reqctx->src_nents + reqctx->aad_nents + + MIN_CCM_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE - + reqctx->b0_len, transhdr_len, op_type)) { atomic_inc(&adap->chcr_stats.fallback); + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); return ERR_PTR(chcr_aead_fallback(req, op_type)); } - - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); + chcr_req = (struct chcr_wr *) __skb_put_zero(skb, transhdr_len); - chcr_req = __skb_put_zero(skb, transhdr_len); - - fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type, ctx); + fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type); chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); @@ -2308,29 +2724,37 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, 16), aeadctx->key, aeadctx->enckey_len); phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); error = ccm_format_packet(req, aeadctx, sub_type, op_type); if (error) goto dstmap_fail; - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); - sg_param.qid = qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) + reqctx->b0_dma = dma_map_single(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, + &reqctx->scratch_pad, reqctx->b0_len, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, + reqctx->b0_dma)) { + error = -ENOMEM; goto dstmap_fail; + } + + chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); + chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); - skb_set_transport_header(skb, transhdr_len); - frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx); atomic_inc(&adap->chcr_stats.aead_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, 0, - sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + + kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen + + reqctx->b0_len) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0, + transhdr_len, temp, 0); reqctx->skb = skb; - skb_get(skb); + reqctx->op = op_type; + return skb; dstmap_fail: kfree_skb(skb); err: + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type); return ERR_PTR(error); } @@ -2340,101 +2764,84 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; - struct phys_sge_parm sg_param; - struct scatterlist *src; - unsigned int frags = 0, transhdr_len; - unsigned int ivsize = AES_BLOCK_SIZE; - unsigned int dst_size = 0, kctx_len, nents, assoclen = req->assoclen; - unsigned char tag_offset = 0; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len, dnents = 0; + unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); - int error = -EINVAL, src_nent; + int error = -EINVAL; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); - - dst_size = assoclen + req->cryptlen + (op_type ? -authsize : - authsize); - /* validate key size */ - if (aeadctx->enckey_len == 0) - goto err; + struct adapter *adap = padap(a_ctx(tfm)->dev); - if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) - goto err; - src_nent = sg_nents_for_len(req->src, assoclen + req->cryptlen); - if (src_nent < 0) - goto err; + if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) + assoclen = req->assoclen - 8; - src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen); - reqctx->dst = src; - if (req->src != req->dst) { - error = chcr_copy_assoc(req, aeadctx); + reqctx->b0_dma = 0; + dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize); + error = chcr_aead_common_init(req, op_type); if (error) return ERR_PTR(error); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, - req->assoclen); - } - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)); - if (reqctx->dst_nents < 0) { - pr_err("GCM:Invalid Destination sg entries\n"); - error = -EINVAL; - goto err; + if (dst_size) { + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, + req->cryptlen + (op_type ? -authsize : authsize), + CHCR_DST_SG_SIZE, req->assoclen); + dnents += MIN_GCM_SG; // For IV + } else { + dnents = 0; } - - nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)) : 0; - dst_size = get_space_for_phys_dsgl(nents); + dst_size = get_space_for_phys_dsgl(dnents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) + AEAD_H_SIZE; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); - if (chcr_aead_need_fallback(req, src_nent + MIN_GCM_SG, - T6_MAX_AAD_SIZE, - transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8), - op_type)) { + reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) <= + SGE_MAX_WR_LEN; + temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + + req->cryptlen), 16) * 16) : (sgl_len(reqctx->src_nents + + reqctx->aad_nents + MIN_GCM_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, + transhdr_len, op_type)) { atomic_inc(&adap->chcr_stats.fallback); + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); return ERR_PTR(chcr_aead_fallback(req, op_type)); } - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - /* NIC driver is going to write the sge hdr. */ - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); - chcr_req = __skb_put_zero(skb, transhdr_len); - if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) - assoclen = req->assoclen - 8; - - tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; + //Offset of tag from end + temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( - ctx->dev->rx_channel_id, 2, (ivsize ? - (assoclen + 1) : 0)); + a_ctx(tfm)->dev->rx_channel_id, 2, + (assoclen + 1)); chcr_req->sec_cpl.pldlen = - htonl(assoclen + ivsize + req->cryptlen); + htonl(assoclen + IV + req->cryptlen); chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( assoclen ? 1 : 0, assoclen, - assoclen + ivsize + 1, 0); + assoclen + IV + 1, 0); chcr_req->sec_cpl.cipherstop_lo_authinsert = - FILL_SEC_CPL_AUTHINSERT(0, assoclen + ivsize + 1, - tag_offset, tag_offset); + FILL_SEC_CPL_AUTHINSERT(0, assoclen + IV + 1, + temp, temp); chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, CHCR_SCMD_CIPHER_MODE_AES_GCM, CHCR_SCMD_AUTH_MODE_GHASH, - aeadctx->hmac_ctrl, ivsize >> 1); + aeadctx->hmac_ctrl, IV >> 1); chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, - 0, 1, dst_size); + 0, 0, dst_size); chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) * @@ -2452,30 +2859,21 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01); phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); - sg_param.qid = qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) - goto dstmap_fail; + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); - skb_set_transport_header(skb, transhdr_len); - write_sg_to_skb(skb, &frags, req->src, assoclen); - write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); - write_sg_to_skb(skb, &frags, src, req->cryptlen); + chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); + chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); atomic_inc(&adap->chcr_stats.aead_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, size, - sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, - reqctx->verify); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + + kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, + transhdr_len, temp, reqctx->verify); reqctx->skb = skb; - skb_get(skb); + reqctx->op = op_type; return skb; -dstmap_fail: - /* ivmap_fail: */ - kfree_skb(skb); err: + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type); return ERR_PTR(error); } @@ -2483,8 +2881,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, static int chcr_aead_cra_init(struct crypto_aead *tfm) { - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct aead_alg *alg = crypto_aead_alg(tfm); aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, @@ -2495,25 +2892,20 @@ static int chcr_aead_cra_init(struct crypto_aead *tfm) crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx), sizeof(struct aead_request) + crypto_aead_reqsize(aeadctx->sw_cipher))); - aeadctx->null = crypto_get_default_null_skcipher(); - if (IS_ERR(aeadctx->null)) - return PTR_ERR(aeadctx->null); - return chcr_device_init(ctx); + return chcr_device_init(a_ctx(tfm)); } static void chcr_aead_cra_exit(struct crypto_aead *tfm) { - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); - crypto_put_default_null_skcipher(); crypto_free_aead(aeadctx->sw_cipher); } static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; aeadctx->mayverify = VERIFY_HW; @@ -2522,7 +2914,7 @@ static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, static int chcr_authenc_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); u32 maxauth = crypto_aead_maxauthsize(tfm); /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not @@ -2560,7 +2952,7 @@ static int chcr_authenc_setauthsize(struct crypto_aead *tfm, static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); switch (authsize) { case ICV_4: @@ -2600,7 +2992,7 @@ static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); switch (authsize) { case ICV_8: @@ -2626,7 +3018,7 @@ static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, static int chcr_ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); switch (authsize) { case ICV_4: @@ -2669,8 +3061,7 @@ static int chcr_ccm_common_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); unsigned char ck_size, mk_size; int key_ctx_size = 0; @@ -2703,8 +3094,7 @@ static int chcr_aead_ccm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); int error; crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); @@ -2722,8 +3112,7 @@ static int chcr_aead_ccm_setkey(struct crypto_aead *aead, static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); int error; if (keylen < 3) { @@ -2749,8 +3138,7 @@ static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx); struct crypto_cipher *cipher; unsigned int ck_size; @@ -2822,8 +3210,7 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(authenc); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); /* it contains auth and cipher key both*/ struct crypto_authenc_keys keys; @@ -2943,8 +3330,7 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(authenc); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); struct crypto_authenc_keys keys; int err; @@ -3016,7 +3402,7 @@ static int chcr_aead_encrypt(struct aead_request *req) static int chcr_aead_decrypt(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); int size; @@ -3049,30 +3435,29 @@ static int chcr_aead_op(struct aead_request *req, create_wr_t create_wr_fn) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); struct uld_ctx *u_ctx; struct sk_buff *skb; - if (!ctx->dev) { + if (!a_ctx(tfm)->dev) { pr_err("chcr : %s : No crypto device.\n", __func__); return -ENXIO; } - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(a_ctx(tfm)); if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx)) { + a_ctx(tfm)->tx_qidx)) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } /* Form a WR from req */ - skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], size, + skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size, op_type); if (IS_ERR(skb) || !skb) return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index d1a787b..96c9335 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -214,27 +214,22 @@ calc_tx_flits_ofld(skb) * 8), 16))) #define FILL_CMD_MORE(immdatalen) htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) |\ - ULP_TX_SC_MORE_V((immdatalen) ? 0 : 1)) - + ULP_TX_SC_MORE_V((immdatalen))) #define MAX_NK 8 -#define CRYPTO_MAX_IMM_TX_PKT_LEN 256 -#define MAX_WR_SIZE 512 #define ROUND_16(bytes) ((bytes) & 0xFFFFFFF0) #define MAX_DSGL_ENT 32 -#define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 1) #define MIN_CIPHER_SG 1 /* IV */ -#define MIN_AUTH_SG 2 /*IV + AAD*/ -#define MIN_GCM_SG 2 /* IV + AAD*/ +#define MIN_AUTH_SG 1 /* IV */ +#define MIN_GCM_SG 1 /* IV */ #define MIN_DIGEST_SG 1 /*Partial Buffer*/ -#define MIN_CCM_SG 3 /*IV+AAD+B0*/ +#define MIN_CCM_SG 2 /*IV+B0*/ #define SPACE_LEFT(len) \ - ((MAX_WR_SIZE - WR_MIN_LEN - (len))) + ((SGE_MAX_WR_LEN - WR_MIN_LEN - (len))) -unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, - 48, 64, 72, 88, - 96, 112, 120, 136, - 144, 160, 168, 184, - 192}; +unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88, + 96, 112, 120, 136, 144, 160, 168, 184, + 192, 208, 216, 232, 240, 256, 264, 280, + 288, 304, 312, 328, 336, 352, 360, 376}; unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80, 112, 112, 128, 128, 144, 144, 160, 160, 192, 192, 208, 208, 224, 224, 240, 240, @@ -258,7 +253,6 @@ struct hash_wr_param { struct cipher_wr_param { struct ablkcipher_request *req; - struct scatterlist *srcsg; char *iv; int bytes; unsigned short qid; @@ -298,31 +292,11 @@ enum { ICV_16 = 16 }; -struct hash_op_params { - unsigned char mk_size; - unsigned char pad_align; - unsigned char auth_mode; - char hash_name[MAX_HASH_NAME]; - unsigned short block_size; - unsigned short word_size; - unsigned short ipad_size; -}; - struct phys_sge_pairs { __be16 len[8]; __be64 addr[8]; }; -struct phys_sge_parm { - unsigned int nents; - unsigned int obsize; - unsigned short qid; -}; - -struct crypto_result { - struct completion completion; - int err; -}; static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 8f436f8..94a87e3 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -149,9 +149,23 @@ #define CHCR_HASH_MAX_BLOCK_SIZE_64 64 #define CHCR_HASH_MAX_BLOCK_SIZE_128 128 -#define CHCR_SG_SIZE 2048 +#define CHCR_SRC_SG_SIZE (0x10000 - sizeof(int)) +#define CHCR_DST_SG_SIZE 2048 -/* Aligned to 128 bit boundary */ +static inline struct chcr_context *a_ctx(struct crypto_aead *tfm) +{ + return crypto_aead_ctx(tfm); +} + +static inline struct chcr_context *c_ctx(struct crypto_ablkcipher *tfm) +{ + return crypto_ablkcipher_ctx(tfm); +} + +static inline struct chcr_context *h_ctx(struct crypto_ahash *tfm) +{ + return crypto_tfm_ctx(crypto_ahash_tfm(tfm)); +} struct ablk_ctx { struct crypto_skcipher *sw_cipher; @@ -165,15 +179,39 @@ struct ablk_ctx { }; struct chcr_aead_reqctx { struct sk_buff *skb; - struct scatterlist *dst; - struct scatterlist srcffwd[2]; - struct scatterlist dstffwd[2]; + dma_addr_t iv_dma; + dma_addr_t b0_dma; + unsigned int b0_len; + unsigned int op; + short int aad_nents; + short int src_nents; short int dst_nents; + u16 imm; u16 verify; u8 iv[CHCR_MAX_CRYPTO_IV_LEN]; unsigned char scratch_pad[MAX_SCRATCH_PAD_SIZE]; }; +struct ulptx_walk { + struct ulptx_sgl *sgl; + unsigned int nents; + unsigned int pair_idx; + unsigned int last_sg_len; + struct scatterlist *last_sg; + struct ulptx_sge_pair *pair; + +}; + +struct dsgl_walk { + unsigned int nents; + unsigned int last_sg_len; + struct scatterlist *last_sg; + struct cpl_rx_phys_dsgl *dsgl; + struct phys_sge_pairs *to; +}; + + + struct chcr_gcm_ctx { u8 ghash_h[AEAD_H_SIZE]; }; @@ -194,7 +232,6 @@ struct __aead_ctx { struct chcr_aead_ctx { __be32 key_ctx_hdr; unsigned int enckey_len; - struct crypto_skcipher *null; struct crypto_aead *sw_cipher; u8 salt[MAX_SALT]; u8 key[CHCR_AES_MAX_KEY_LEN]; @@ -230,8 +267,11 @@ struct chcr_ahash_req_ctx { u8 bfr2[CHCR_HASH_MAX_BLOCK_SIZE_128]; u8 *reqbfr; u8 *skbfr; + dma_addr_t dma_addr; + u32 dma_len; u8 reqlen; - /* DMA the partial hash in it */ + u8 imm; + u8 is_sg_map; u8 partial_hash[CHCR_HASH_MAX_DIGEST_SIZE]; u64 data_len; /* Data len till time */ /* SKB which is being sent to the hardware for processing */ @@ -240,14 +280,15 @@ struct chcr_ahash_req_ctx { struct chcr_blkcipher_req_ctx { struct sk_buff *skb; - struct scatterlist srcffwd[2]; - struct scatterlist dstffwd[2]; struct scatterlist *dstsg; - struct scatterlist *dst; unsigned int processed; unsigned int last_req_len; + struct scatterlist *srcsg; + unsigned int src_ofst; + unsigned int dst_ofst; unsigned int op; - short int dst_nents; + dma_addr_t iv_dma; + u16 imm; u8 iv[CHCR_MAX_CRYPTO_IV_LEN]; }; @@ -261,24 +302,6 @@ struct chcr_alg_template { } alg; }; -struct chcr_req_ctx { - union { - struct ahash_request *ahash_req; - struct aead_request *aead_req; - struct ablkcipher_request *ablk_req; - } req; - union { - struct chcr_ahash_req_ctx *ahash_ctx; - struct chcr_aead_reqctx *reqctx; - struct chcr_blkcipher_req_ctx *ablk_ctx; - } ctx; -}; - -struct sge_opaque_hdr { - void *dev; - dma_addr_t addr[MAX_SKB_FRAGS + 1]; -}; - typedef struct sk_buff *(*create_wr_t)(struct aead_request *req, unsigned short qid, int size, @@ -291,4 +314,37 @@ static int chcr_aead_op(struct aead_request *req_base, static inline int get_aead_subtype(struct crypto_aead *aead); static int chcr_handle_cipher_resp(struct ablkcipher_request *req, unsigned char *input, int err); +static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err); +static int chcr_aead_dma_map(struct device *dev, struct aead_request *req, + unsigned short op_type); +static void chcr_aead_dma_unmap(struct device *dev, struct aead_request + *req, unsigned short op_type); +static inline void chcr_add_aead_dst_ent(struct aead_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + unsigned int assoclen, + unsigned short op_type, + unsigned short qid); +static inline void chcr_add_aead_src_ent(struct aead_request *req, + struct ulptx_sgl *ulptx, + unsigned int assoclen, + unsigned short op_type); +static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, + struct ulptx_sgl *ulptx, + struct cipher_wr_param *wrparam); +static int chcr_cipher_dma_map(struct device *dev, + struct ablkcipher_request *req); +static void chcr_cipher_dma_unmap(struct device *dev, + struct ablkcipher_request *req); +static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + struct cipher_wr_param *wrparam, + unsigned short qid); +int sg_nents_len_skip(struct scatterlist *sg, u64 len, u64 skip); +static inline void chcr_add_hash_src_ent(struct ahash_request *req, + struct ulptx_sgl *ulptx, + struct hash_wr_param *param); +static inline int chcr_hash_dma_map(struct device *dev, + struct ahash_request *req); +static inline void chcr_hash_dma_unmap(struct device *dev, + struct ahash_request *req); #endif /* __CHCR_CRYPTO_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c index ede1220..1c1cc48 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -1537,7 +1537,13 @@ int t4_mgmt_tx(struct adapter *adap, struct sk_buff *skb) */ static inline int is_ofld_imm(const struct sk_buff *skb) { - return skb->len <= MAX_IMM_TX_PKT_LEN; + struct work_request_hdr *req = (struct work_request_hdr *)skb->data; + unsigned long opcode = FW_WR_OP_G(ntohl(req->wr_hi)); + + if (opcode == FW_CRYPTO_LOOKASIDE_WR) + return skb->len <= SGE_MAX_WR_LEN; + else + return skb->len <= MAX_IMM_TX_PKT_LEN; } /** -- 2.1.4 ^ permalink raw reply related [flat|nested] 18+ messages in thread
* [PATCH 7/7] crypto:chelsio: Fix memory leak 2017-10-03 6:46 [PATCH 0/7]crypto:chelsio: Bugs fixes Harsh Jain ` (5 preceding siblings ...) 2017-10-03 6:46 ` [PATCH 6/7] crypto:chelsio:Move DMA un/mapping to chcr from lld cxgb4 driver Harsh Jain @ 2017-10-03 6:46 ` Harsh Jain [not found] ` <cover.1507449374.git.harsh@chelsio.com> 7 siblings, 0 replies; 18+ messages in thread From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw) To: herbert, linux-crypto, netdev; +Cc: Harsh Jain Fix memory leak when device does not support crypto. Reported-by: Dan Carpenter <dan.carpenter@oracle.com> Signed-off-by: Harsh Jain <harsh@chelsio.com> --- drivers/crypto/chelsio/chcr_core.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index b6dd9cb..4f677b3 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -154,15 +154,15 @@ static void *chcr_uld_add(const struct cxgb4_lld_info *lld) struct uld_ctx *u_ctx; /* Create the device and add it in the device list */ + if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE)) + return ERR_PTR(-EOPNOTSUPP); + + /* Create the device and add it in the device list */ u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL); if (!u_ctx) { u_ctx = ERR_PTR(-ENOMEM); goto out; } - if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE)) { - u_ctx = ERR_PTR(-ENOMEM); - goto out; - } u_ctx->lldi = *lld; out: return u_ctx; -- 2.1.4 ^ permalink raw reply related [flat|nested] 18+ messages in thread
[parent not found: <cover.1507449374.git.harsh@chelsio.com>]
* [PATCH v2 1/7] crypto:chelsio: Remove unused parameter [not found] ` <cover.1507449374.git.harsh@chelsio.com> @ 2017-10-08 8:07 ` Harsh Jain 2017-11-03 14:21 ` Herbert Xu 2017-10-08 8:07 ` [PATCH v2 2/7] crypto:chelsio: Check error code with IS_ERR macro Harsh Jain ` (5 subsequent siblings) 6 siblings, 1 reply; 18+ messages in thread From: Harsh Jain @ 2017-10-08 8:07 UTC (permalink / raw) To: herbert, linux-crypto, netdev; +Cc: Yeshaswi M R Gowda, Harsh Jain From: Yeshaswi M R Gowda <yeshaswi@chelsio.com> Remove unused parameter sent to latest fw. Signed-off-by: Harsh Jain <harsh@chelsio.com> --- drivers/crypto/chelsio/chcr_algo.c | 43 +++++++++++++++----------------------- drivers/crypto/chelsio/chcr_algo.h | 12 +++++------ 2 files changed, 23 insertions(+), 32 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 0e81607..bdb1014 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -577,36 +577,27 @@ static int chcr_cipher_fallback(struct crypto_skcipher *cipher, static inline void create_wreq(struct chcr_context *ctx, struct chcr_wr *chcr_req, void *req, struct sk_buff *skb, - int kctx_len, int hash_sz, - int is_iv, + int hash_sz, unsigned int sc_len, unsigned int lcb) { struct uld_ctx *u_ctx = ULD_CTX(ctx); - int iv_loc = IV_DSGL; int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; - unsigned int immdatalen = 0, nr_frags = 0; + unsigned int immdatalen = 0; - if (is_ofld_imm(skb)) { + if (is_ofld_imm(skb)) immdatalen = skb->data_len; - iv_loc = IV_IMMEDIATE; - } else { - nr_frags = skb_shinfo(skb)->nr_frags; - } - chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen, - ((sizeof(chcr_req->key_ctx) + kctx_len) >> 4)); + chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE; chcr_req->wreq.pld_size_hash_size = - htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) | - FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); + htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); chcr_req->wreq.len16_pkd = htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP( (calc_tx_flits_ofld(skb) * 8), 16))); chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid, - is_iv ? iv_loc : IV_NOP, !!lcb, - ctx->tx_qidx); + !!lcb, ctx->tx_qidx); chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id, qid); @@ -616,7 +607,7 @@ static inline void create_wreq(struct chcr_context *ctx, chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen); chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + sizeof(chcr_req->key_ctx) + - kctx_len + sc_len + immdatalen); + sc_len + immdatalen); } /** @@ -706,8 +697,8 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes); atomic_inc(&adap->chcr_stats.cipher_rqst); - create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, kctx_len, 0, 1, - sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl, + create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, 0, + sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl + kctx_len, ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); reqctx->skb = skb; skb_get(skb); @@ -1417,8 +1408,8 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, if (param->sg_len != 0) write_sg_to_skb(skb, &frags, req->src, param->sg_len); atomic_inc(&adap->chcr_stats.digest_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, - hash_size_in_response, 0, DUMMY_BYTES, 0); + create_wreq(ctx, chcr_req, &req->base, skb, hash_size_in_response, + DUMMY_BYTES + kctx_len, 0); req_ctx->skb = skb; skb_get(skb); return skb; @@ -2080,8 +2071,8 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, write_buffer_to_skb(skb, &frags, req->iv, ivsize); write_sg_to_skb(skb, &frags, src, req->cryptlen); atomic_inc(&adap->chcr_stats.cipher_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); + create_wreq(ctx, chcr_req, &req->base, skb, size, + sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0); reqctx->skb = skb; skb_get(skb); @@ -2396,8 +2387,8 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, skb_set_transport_header(skb, transhdr_len); frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx); atomic_inc(&adap->chcr_stats.aead_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, 0, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); + create_wreq(ctx, chcr_req, &req->base, skb, 0, + sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0); reqctx->skb = skb; skb_get(skb); return skb; @@ -2554,8 +2545,8 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); write_sg_to_skb(skb, &frags, src, req->cryptlen); atomic_inc(&adap->chcr_stats.aead_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size, + create_wreq(ctx, chcr_req, &req->base, skb, size, + sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, reqctx->verify); reqctx->skb = skb; skb_get(skb); diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index 583008d..54851ec 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -176,21 +176,21 @@ KEY_CONTEXT_SALT_PRESENT_V(1) | \ KEY_CONTEXT_CTX_LEN_V((ctx_len))) -#define FILL_WR_OP_CCTX_SIZE(len, ctx_len) \ +#define FILL_WR_OP_CCTX_SIZE \ htonl( \ FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \ FW_CRYPTO_LOOKASIDE_WR) | \ FW_CRYPTO_LOOKASIDE_WR_COMPL_V(0) | \ - FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V((len)) | \ - FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(1) | \ - FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V((ctx_len))) + FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V((0)) | \ + FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(0) | \ + FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V(0)) -#define FILL_WR_RX_Q_ID(cid, qid, wr_iv, lcb, fid) \ +#define FILL_WR_RX_Q_ID(cid, qid, lcb, fid) \ htonl( \ FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V((cid)) | \ FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V((qid)) | \ FW_CRYPTO_LOOKASIDE_WR_LCB_V((lcb)) | \ - FW_CRYPTO_LOOKASIDE_WR_IV_V((wr_iv)) | \ + FW_CRYPTO_LOOKASIDE_WR_IV_V((IV_NOP)) | \ FW_CRYPTO_LOOKASIDE_WR_FQIDX_V(fid)) #define FILL_ULPTX_CMD_DEST(cid, qid) \ -- 2.1.4 ^ permalink raw reply related [flat|nested] 18+ messages in thread
* Re: [PATCH v2 1/7] crypto:chelsio: Remove unused parameter 2017-10-08 8:07 ` [PATCH v2 1/7] crypto:chelsio: Remove unused parameter Harsh Jain @ 2017-11-03 14:21 ` Herbert Xu 0 siblings, 0 replies; 18+ messages in thread From: Herbert Xu @ 2017-11-03 14:21 UTC (permalink / raw) To: Harsh Jain; +Cc: linux-crypto, netdev, Yeshaswi M R Gowda On Sun, Oct 08, 2017 at 01:37:18PM +0530, Harsh Jain wrote: > From: Yeshaswi M R Gowda <yeshaswi@chelsio.com> > > Remove unused parameter sent to latest fw. > > Signed-off-by: Harsh Jain <harsh@chelsio.com> All applied. Thanks. -- Email: Herbert Xu <herbert@gondor.apana.org.au> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt ^ permalink raw reply [flat|nested] 18+ messages in thread
* [PATCH v2 2/7] crypto:chelsio: Check error code with IS_ERR macro [not found] ` <cover.1507449374.git.harsh@chelsio.com> 2017-10-08 8:07 ` [PATCH v2 1/7] crypto:chelsio: Remove unused parameter Harsh Jain @ 2017-10-08 8:07 ` Harsh Jain 2017-10-08 8:07 ` [PATCH v2 3/7] crypto:gf128mul: The x8_ble multiplication functions Harsh Jain ` (4 subsequent siblings) 6 siblings, 0 replies; 18+ messages in thread From: Harsh Jain @ 2017-10-08 8:07 UTC (permalink / raw) To: herbert, linux-crypto, netdev Cc: Yeshaswi M R Gowda, Jitendra Lulla, Harsh Jain From: Yeshaswi M R Gowda <yeshaswi@chelsio.com> Check and return proper error code. Signed-off-by: Jitendra Lulla <jlulla@chelsio.com> Signed-off-by: Harsh Jain <harsh@chelsio.com> --- drivers/crypto/chelsio/chcr_algo.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index bdb1014..e4bf32d 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -1455,8 +1455,8 @@ static int chcr_ahash_update(struct ahash_request *req) req_ctx->result = 0; req_ctx->data_len += params.sg_len + params.bfr_len; skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; + if (IS_ERR(skb)) + return PTR_ERR(skb); if (remainder) { u8 *temp; @@ -1519,8 +1519,8 @@ static int chcr_ahash_final(struct ahash_request *req) params.more = 0; } skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; + if (IS_ERR(skb)) + return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); @@ -1570,8 +1570,8 @@ static int chcr_ahash_finup(struct ahash_request *req) } skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; + if (IS_ERR(skb)) + return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); @@ -1621,8 +1621,8 @@ static int chcr_ahash_digest(struct ahash_request *req) } skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; + if (IS_ERR(skb)) + return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); -- 2.1.4 ^ permalink raw reply related [flat|nested] 18+ messages in thread
* [PATCH v2 3/7] crypto:gf128mul: The x8_ble multiplication functions [not found] ` <cover.1507449374.git.harsh@chelsio.com> 2017-10-08 8:07 ` [PATCH v2 1/7] crypto:chelsio: Remove unused parameter Harsh Jain 2017-10-08 8:07 ` [PATCH v2 2/7] crypto:chelsio: Check error code with IS_ERR macro Harsh Jain @ 2017-10-08 8:07 ` Harsh Jain 2017-10-08 8:07 ` [PATCH v2 4/7] crypto:chelsio:Use x8_ble gf multiplication to calculate IV Harsh Jain ` (3 subsequent siblings) 6 siblings, 0 replies; 18+ messages in thread From: Harsh Jain @ 2017-10-08 8:07 UTC (permalink / raw) To: herbert, linux-crypto, netdev; +Cc: Harsh Jain It multiply GF(2^128) elements in the ble format. It will be used by chelsio driver to speed up gf multiplication. Signed-off-by: Harsh Jain <harsh@chelsio.com> --- crypto/gf128mul.c | 13 +++++++++++++ include/crypto/gf128mul.h | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c index dc01212..24e6019 100644 --- a/crypto/gf128mul.c +++ b/crypto/gf128mul.c @@ -156,6 +156,19 @@ static void gf128mul_x8_bbe(be128 *x) x->b = cpu_to_be64((b << 8) ^ _tt); } +void gf128mul_x8_ble(le128 *r, const le128 *x) +{ + u64 a = le64_to_cpu(x->a); + u64 b = le64_to_cpu(x->b); + + /* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */ + u64 _tt = gf128mul_table_be[a >> 56]; + + r->a = cpu_to_le64((a << 8) | (b >> 56)); + r->b = cpu_to_le64((b << 8) ^ _tt); +} +EXPORT_SYMBOL(gf128mul_x8_ble); + void gf128mul_lle(be128 *r, const be128 *b) { be128 p[8]; diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h index 0977fb1..fa0a63d 100644 --- a/include/crypto/gf128mul.h +++ b/include/crypto/gf128mul.h @@ -227,7 +227,7 @@ struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g); struct gf128mul_4k *gf128mul_init_4k_bbe(const be128 *g); void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t); void gf128mul_4k_bbe(be128 *a, const struct gf128mul_4k *t); - +void gf128mul_x8_ble(le128 *r, const le128 *x); static inline void gf128mul_free_4k(struct gf128mul_4k *t) { kzfree(t); -- 2.1.4 ^ permalink raw reply related [flat|nested] 18+ messages in thread
* [PATCH v2 4/7] crypto:chelsio:Use x8_ble gf multiplication to calculate IV. [not found] ` <cover.1507449374.git.harsh@chelsio.com> ` (2 preceding siblings ...) 2017-10-08 8:07 ` [PATCH v2 3/7] crypto:gf128mul: The x8_ble multiplication functions Harsh Jain @ 2017-10-08 8:07 ` Harsh Jain 2017-10-08 8:07 ` [PATCH v2 5/7] crypto:chelsio:Remove allocation of sg list to implement 2K limit of dsgl header Harsh Jain ` (2 subsequent siblings) 6 siblings, 0 replies; 18+ messages in thread From: Harsh Jain @ 2017-10-08 8:07 UTC (permalink / raw) To: herbert, linux-crypto, netdev; +Cc: Harsh Jain gf128mul_x8_ble() will reduce gf Multiplication iteration by 8. Signed-off-by: Harsh Jain <harsh@chelsio.com> --- drivers/crypto/chelsio/chcr_algo.c | 11 +++++++++-- drivers/crypto/chelsio/chcr_crypto.h | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index e4bf32d..e0ab34a 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -888,9 +888,11 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) int ret, i; u8 *key; unsigned int keylen; + int round = reqctx->last_req_len / AES_BLOCK_SIZE; + int round8 = round / 8; cipher = ablkctx->aes_generic; - memcpy(iv, req->info, AES_BLOCK_SIZE); + memcpy(iv, reqctx->iv, AES_BLOCK_SIZE); keylen = ablkctx->enckey_len / 2; key = ablkctx->key + keylen; @@ -899,7 +901,10 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) goto out; crypto_cipher_encrypt_one(cipher, iv, iv); - for (i = 0; i < (reqctx->processed / AES_BLOCK_SIZE); i++) + for (i = 0; i < round8; i++) + gf128mul_x8_ble((le128 *)iv, (le128 *)iv); + + for (i = 0; i < (round % 8); i++) gf128mul_x_ble((le128 *)iv, (le128 *)iv); crypto_cipher_decrypt_one(cipher, iv, iv); @@ -1040,6 +1045,7 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, CRYPTO_ALG_SUB_TYPE_CTR) bytes = adjust_ctr_overflow(reqctx->iv, bytes); reqctx->processed += bytes; + reqctx->last_req_len = bytes; wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; wrparam.req = req; wrparam.bytes = bytes; @@ -1132,6 +1138,7 @@ static int process_cipher(struct ablkcipher_request *req, goto error; } reqctx->processed = bytes; + reqctx->last_req_len = bytes; reqctx->dst = reqctx->dstsg; reqctx->op = op_type; wrparam.qid = qid; diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 30af1ee..b3722b3 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -247,6 +247,7 @@ struct chcr_blkcipher_req_ctx { struct scatterlist *dst; struct scatterlist *newdstsg; unsigned int processed; + unsigned int last_req_len; unsigned int op; short int dst_nents; u8 iv[CHCR_MAX_CRYPTO_IV_LEN]; -- 2.1.4 ^ permalink raw reply related [flat|nested] 18+ messages in thread
* [PATCH v2 5/7] crypto:chelsio:Remove allocation of sg list to implement 2K limit of dsgl header [not found] ` <cover.1507449374.git.harsh@chelsio.com> ` (3 preceding siblings ...) 2017-10-08 8:07 ` [PATCH v2 4/7] crypto:chelsio:Use x8_ble gf multiplication to calculate IV Harsh Jain @ 2017-10-08 8:07 ` Harsh Jain 2017-10-08 8:07 ` [PATCH v2 6/7] crypto:chelsio:Move DMA un/mapping to chcr from lld cxgb4 driver Harsh Jain 2017-10-08 8:07 ` [PATCH v2 7/7] crypto:chelsio: Fix memory leak Harsh Jain 6 siblings, 0 replies; 18+ messages in thread From: Harsh Jain @ 2017-10-08 8:07 UTC (permalink / raw) To: herbert, linux-crypto, netdev; +Cc: Harsh Jain Update DMA address index instead of allocating new sg list to impose 2k size limit for each entry. Signed-off-by: Harsh Jain <harsh@chelsio.com> --- drivers/crypto/chelsio/chcr_algo.c | 237 +++++++++++------------------------ drivers/crypto/chelsio/chcr_algo.h | 3 +- drivers/crypto/chelsio/chcr_core.h | 2 +- drivers/crypto/chelsio/chcr_crypto.h | 6 - 4 files changed, 76 insertions(+), 172 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index e0ab34a..b13991d 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -117,6 +117,21 @@ static inline unsigned int sgl_len(unsigned int n) return (3 * n) / 2 + (n & 1) + 2; } +static int dstsg_2k(struct scatterlist *sgl, unsigned int reqlen) +{ + int nents = 0; + unsigned int less; + + while (sgl && reqlen) { + less = min(reqlen, sgl->length); + nents += DIV_ROUND_UP(less, CHCR_SG_SIZE); + reqlen -= less; + sgl = sg_next(sgl); + } + + return nents; +} + static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) { u8 temp[SHA512_DIGEST_SIZE]; @@ -166,8 +181,6 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, kfree_skb(ctx_req.ctx.reqctx->skb); ctx_req.ctx.reqctx->skb = NULL; } - free_new_sg(ctx_req.ctx.reqctx->newdstsg); - ctx_req.ctx.reqctx->newdstsg = NULL; if (ctx_req.ctx.reqctx->verify == VERIFY_SW) { chcr_verify_tag(ctx_req.req.aead_req, input, &err); @@ -388,31 +401,41 @@ static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl, { struct phys_sge_pairs *to; unsigned int len = 0, left_size = sg_param->obsize; - unsigned int nents = sg_param->nents, i, j = 0; + unsigned int j = 0; + int offset, ent_len; phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); + to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl + + sizeof(struct cpl_rx_phys_dsgl)); + while (left_size && sg) { + len = min_t(u32, left_size, sg_dma_len(sg)); + offset = 0; + while (len) { + ent_len = min_t(u32, len, CHCR_SG_SIZE); + to->len[j % 8] = htons(ent_len); + to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + + offset); + offset += ent_len; + len -= ent_len; + j++; + if ((j % 8) == 0) + to++; + } + left_size -= min(left_size, sg_dma_len(sg)); + sg = sg_next(sg); + } phys_cpl->pcirlxorder_to_noofsgentr = htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | CPL_RX_PHYS_DSGL_DCAID_V(0) | - CPL_RX_PHYS_DSGL_NOOFSGENTR_V(nents)); + CPL_RX_PHYS_DSGL_NOOFSGENTR_V(j)); phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; phys_cpl->rss_hdr_int.qid = htons(sg_param->qid); phys_cpl->rss_hdr_int.hash_val = 0; - to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl + - sizeof(struct cpl_rx_phys_dsgl)); - for (i = 0; nents && left_size; to++) { - for (j = 0; j < 8 && nents && left_size; j++, nents--) { - len = min(left_size, sg_dma_len(sg)); - to->len[j] = htons(len); - to->addr[j] = cpu_to_be64(sg_dma_address(sg)); - left_size -= len; - sg = sg_next(sg); - } - } + } static inline int map_writesg_phys_cpl(struct device *dev, @@ -523,31 +546,33 @@ static int generate_copy_rrkey(struct ablk_ctx *ablkctx, static int chcr_sg_ent_in_wr(struct scatterlist *src, struct scatterlist *dst, unsigned int minsg, - unsigned int space, - short int *sent, - short int *dent) + unsigned int space) { int srclen = 0, dstlen = 0; int srcsg = minsg, dstsg = 0; + int offset = 0, less; - *sent = 0; - *dent = 0; while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) && space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { srclen += src->length; srcsg++; + offset = 0; while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { if (srclen <= dstlen) break; - dstlen += dst->length; - dst = sg_next(dst); + less = min_t(unsigned int, dst->length - offset, + CHCR_SG_SIZE); + dstlen += less; + offset += less; + if (offset == dst->length) { + dst = sg_next(dst); + offset = 0; + } dstsg++; } src = sg_next(src); } - *sent = srcsg - minsg; - *dent = dstsg; return min(srclen, dstlen); } @@ -631,13 +656,15 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) struct phys_sge_parm sg_param; unsigned int frags = 0, transhdr_len, phys_dsgl; int error; + int nents; unsigned int ivsize = AES_BLOCK_SIZE, kctx_len; gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; struct adapter *adap = padap(ctx->dev); - phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents); - + reqctx->dst_nents = sg_nents_for_len(reqctx->dst, wrparam->bytes); + nents = dstsg_2k(reqctx->dst, wrparam->bytes); + phys_dsgl = get_space_for_phys_dsgl(nents); kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); @@ -1020,8 +1047,7 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, goto complete; } bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1, - SPACE_LEFT(ablkctx->enckey_len), - &wrparam.snent, &reqctx->dst_nents); + SPACE_LEFT(ablkctx->enckey_len)); if ((bytes + reqctx->processed) >= req->nbytes) bytes = req->nbytes - reqctx->processed; else @@ -1060,8 +1086,6 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, chcr_send_wr(skb); return 0; complete: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; req->base.complete(&req->base, err); return err; } @@ -1077,9 +1101,8 @@ static int process_cipher(struct ablkcipher_request *req, struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); struct ablk_ctx *ablkctx = ABLK_CTX(ctx); struct cipher_wr_param wrparam; - int bytes, nents, err = -EINVAL; + int bytes, err = -EINVAL; - reqctx->newdstsg = NULL; reqctx->processed = 0; if (!req->info) goto error; @@ -1091,18 +1114,9 @@ static int process_cipher(struct ablkcipher_request *req, goto error; } wrparam.srcsg = req->src; - if (is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return PTR_ERR(reqctx->newdstsg); - reqctx->dstsg = reqctx->newdstsg; - } else { reqctx->dstsg = req->dst; - } bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG, - SPACE_LEFT(ablkctx->enckey_len), - &wrparam.snent, - &reqctx->dst_nents); + SPACE_LEFT(ablkctx->enckey_len)); if ((bytes + reqctx->processed) >= req->nbytes) bytes = req->nbytes - reqctx->processed; else @@ -1152,8 +1166,6 @@ static int process_cipher(struct ablkcipher_request *req, return 0; error: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return err; } @@ -1824,63 +1836,6 @@ static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) } } -static int is_newsg(struct scatterlist *sgl, unsigned int *newents) -{ - int nents = 0; - int ret = 0; - - while (sgl) { - if (sgl->length > CHCR_SG_SIZE) - ret = 1; - nents += DIV_ROUND_UP(sgl->length, CHCR_SG_SIZE); - sgl = sg_next(sgl); - } - *newents = nents; - return ret; -} - -static inline void free_new_sg(struct scatterlist *sgl) -{ - kfree(sgl); -} - -static struct scatterlist *alloc_new_sg(struct scatterlist *sgl, - unsigned int nents) -{ - struct scatterlist *newsg, *sg; - int i, len, processed = 0; - struct page *spage; - int offset; - - newsg = kmalloc_array(nents, sizeof(struct scatterlist), GFP_KERNEL); - if (!newsg) - return ERR_PTR(-ENOMEM); - sg = newsg; - sg_init_table(sg, nents); - offset = sgl->offset; - spage = sg_page(sgl); - for (i = 0; i < nents; i++) { - len = min_t(u32, sgl->length - processed, CHCR_SG_SIZE); - sg_set_page(sg, spage, len, offset); - processed += len; - offset += len; - if (offset >= PAGE_SIZE) { - offset = offset % PAGE_SIZE; - spage++; - } - if (processed == sgl->length) { - processed = 0; - sgl = sg_next(sgl); - if (!sgl) - break; - spage = sg_page(sgl); - offset = sgl->offset; - } - sg = sg_next(sg); - } - return newsg; -} - static int chcr_copy_assoc(struct aead_request *req, struct chcr_aead_ctx *ctx) { @@ -1953,7 +1908,6 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, GFP_ATOMIC; struct adapter *adap = padap(ctx->dev); - reqctx->newdstsg = NULL; dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : authsize); if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0)) @@ -1965,24 +1919,13 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, if (src_nent < 0) goto err; src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); - + reqctx->dst = src; if (req->src != req->dst) { error = chcr_copy_assoc(req, aeadctx); if (error) return ERR_PTR(error); - } - if (dst_size && is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return ERR_CAST(reqctx->newdstsg); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - reqctx->newdstsg, req->assoclen); - } else { - if (req->src == req->dst) - reqctx->dst = src; - else - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - req->dst, req->assoclen); + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, + req->assoclen); } if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) { null = 1; @@ -1995,7 +1938,9 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, error = -EINVAL; goto err; } - dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); + nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + + (op_type ? -authsize : authsize)) : 0; + dst_size = get_space_for_phys_dsgl(nents); kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) - sizeof(chcr_req->key_ctx); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); @@ -2004,8 +1949,6 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8), op_type)) { atomic_inc(&adap->chcr_stats.fallback); - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(chcr_aead_fallback(req, op_type)); } skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); @@ -2088,8 +2031,6 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, /* ivmap_fail: */ kfree_skb(skb); err: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(error); } @@ -2307,7 +2248,6 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : authsize); - reqctx->newdstsg = NULL; if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) goto err; src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); @@ -2316,25 +2256,15 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, sub_type = get_aead_subtype(tfm); src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); + reqctx->dst = src; if (req->src != req->dst) { error = chcr_copy_assoc(req, aeadctx); if (error) { pr_err("AAD copy to destination buffer fails\n"); return ERR_PTR(error); } - } - if (dst_size && is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return ERR_CAST(reqctx->newdstsg); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - reqctx->newdstsg, req->assoclen); - } else { - if (req->src == req->dst) - reqctx->dst = src; - else - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - req->dst, req->assoclen); + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, + req->assoclen); } reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + (op_type ? -authsize : authsize)); @@ -2346,8 +2276,9 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type); if (error) goto err; - - dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); + nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + + (op_type ? -authsize : authsize)) : 0; + dst_size = get_space_for_phys_dsgl(nents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG, @@ -2355,8 +2286,6 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8), op_type)) { atomic_inc(&adap->chcr_stats.fallback); - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(chcr_aead_fallback(req, op_type)); } @@ -2402,8 +2331,6 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, dstmap_fail: kfree_skb(skb); err: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(error); } @@ -2432,7 +2359,6 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, GFP_ATOMIC; struct adapter *adap = padap(ctx->dev); - reqctx->newdstsg = NULL; dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize); /* validate key size */ @@ -2446,26 +2372,14 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, goto err; src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen); + reqctx->dst = src; if (req->src != req->dst) { error = chcr_copy_assoc(req, aeadctx); if (error) return ERR_PTR(error); + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, + req->assoclen); } - - if (dst_size && is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return ERR_CAST(reqctx->newdstsg); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - reqctx->newdstsg, assoclen); - } else { - if (req->src == req->dst) - reqctx->dst = src; - else - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - req->dst, assoclen); - } - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + (op_type ? -authsize : authsize)); if (reqctx->dst_nents < 0) { @@ -2474,8 +2388,9 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, goto err; } - - dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); + nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + + (op_type ? -authsize : authsize)) : 0; + dst_size = get_space_for_phys_dsgl(nents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) + AEAD_H_SIZE; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); @@ -2484,8 +2399,6 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8), op_type)) { atomic_inc(&adap->chcr_stats.fallback); - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(chcr_aead_fallback(req, op_type)); } skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); @@ -2563,8 +2476,6 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, /* ivmap_fail: */ kfree_skb(skb); err: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(error); } diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index 54851ec..d1a787b 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -221,7 +221,7 @@ #define MAX_WR_SIZE 512 #define ROUND_16(bytes) ((bytes) & 0xFFFFFFF0) #define MAX_DSGL_ENT 32 -#define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 2) +#define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 1) #define MIN_CIPHER_SG 1 /* IV */ #define MIN_AUTH_SG 2 /*IV + AAD*/ #define MIN_GCM_SG 2 /* IV + AAD*/ @@ -261,7 +261,6 @@ struct cipher_wr_param { struct scatterlist *srcsg; char *iv; int bytes; - short int snent; unsigned short qid; }; enum { diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h index c9a19b2..94e7412 100644 --- a/drivers/crypto/chelsio/chcr_core.h +++ b/drivers/crypto/chelsio/chcr_core.h @@ -89,7 +89,7 @@ struct uld_ctx { struct chcr_dev *dev; }; -struct uld_ctx * assign_chcr_device(void); +struct uld_ctx *assign_chcr_device(void); int chcr_send_wr(struct sk_buff *skb); int start_crypto(void); int stop_crypto(void); diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index b3722b3..8f436f8 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -166,7 +166,6 @@ struct ablk_ctx { struct chcr_aead_reqctx { struct sk_buff *skb; struct scatterlist *dst; - struct scatterlist *newdstsg; struct scatterlist srcffwd[2]; struct scatterlist dstffwd[2]; short int dst_nents; @@ -245,7 +244,6 @@ struct chcr_blkcipher_req_ctx { struct scatterlist dstffwd[2]; struct scatterlist *dstsg; struct scatterlist *dst; - struct scatterlist *newdstsg; unsigned int processed; unsigned int last_req_len; unsigned int op; @@ -291,10 +289,6 @@ static int chcr_aead_op(struct aead_request *req_base, int size, create_wr_t create_wr_fn); static inline int get_aead_subtype(struct crypto_aead *aead); -static int is_newsg(struct scatterlist *sgl, unsigned int *newents); -static struct scatterlist *alloc_new_sg(struct scatterlist *sgl, - unsigned int nents); -static inline void free_new_sg(struct scatterlist *sgl); static int chcr_handle_cipher_resp(struct ablkcipher_request *req, unsigned char *input, int err); #endif /* __CHCR_CRYPTO_H__ */ -- 2.1.4 ^ permalink raw reply related [flat|nested] 18+ messages in thread
* [PATCH v2 6/7] crypto:chelsio:Move DMA un/mapping to chcr from lld cxgb4 driver [not found] ` <cover.1507449374.git.harsh@chelsio.com> ` (4 preceding siblings ...) 2017-10-08 8:07 ` [PATCH v2 5/7] crypto:chelsio:Remove allocation of sg list to implement 2K limit of dsgl header Harsh Jain @ 2017-10-08 8:07 ` Harsh Jain 2017-10-08 8:07 ` [PATCH v2 7/7] crypto:chelsio: Fix memory leak Harsh Jain 6 siblings, 0 replies; 18+ messages in thread From: Harsh Jain @ 2017-10-08 8:07 UTC (permalink / raw) To: herbert, linux-crypto, netdev; +Cc: Harsh Jain Allow chcr to do DMA mapping/Unmapping instead of lld cxgb4. It moves "Copy AAD to dst buffer" requirement from driver to firmware. Signed-off-by: Harsh Jain <harsh@chelsio.com> --- drivers/crypto/chelsio/chcr_algo.c | 1645 ++++++++++++++++++------------ drivers/crypto/chelsio/chcr_algo.h | 44 +- drivers/crypto/chelsio/chcr_crypto.h | 114 ++- drivers/net/ethernet/chelsio/cxgb4/sge.c | 8 +- 4 files changed, 1116 insertions(+), 695 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index b13991d..646dfff 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -70,6 +70,8 @@ #include "chcr_algo.h" #include "chcr_crypto.h" +#define IV AES_BLOCK_SIZE + static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) { return ctx->crypto_ctx->aeadctx; @@ -102,7 +104,7 @@ static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx) static inline int is_ofld_imm(const struct sk_buff *skb) { - return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN); + return (skb->len <= SGE_MAX_WR_LEN); } /* @@ -117,21 +119,92 @@ static inline unsigned int sgl_len(unsigned int n) return (3 * n) / 2 + (n & 1) + 2; } -static int dstsg_2k(struct scatterlist *sgl, unsigned int reqlen) +static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen, + unsigned int entlen, + unsigned int skip) { int nents = 0; unsigned int less; + unsigned int skip_len = 0; - while (sgl && reqlen) { - less = min(reqlen, sgl->length); - nents += DIV_ROUND_UP(less, CHCR_SG_SIZE); - reqlen -= less; - sgl = sg_next(sgl); + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; + sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; + } } + while (sg && reqlen) { + less = min(reqlen, sg_dma_len(sg) - skip_len); + nents += DIV_ROUND_UP(less, entlen); + reqlen -= less; + skip_len = 0; + sg = sg_next(sg); + } return nents; } +static inline void chcr_handle_ahash_resp(struct ahash_request *req, + unsigned char *input, + int err) +{ + struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); + int digestsize, updated_digestsize; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); + + if (input == NULL) + goto out; + reqctx = ahash_request_ctx(req); + digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); + if (reqctx->is_sg_map) + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + if (reqctx->dma_addr) + dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->dma_addr, + reqctx->dma_len, DMA_TO_DEVICE); + reqctx->dma_addr = 0; + updated_digestsize = digestsize; + if (digestsize == SHA224_DIGEST_SIZE) + updated_digestsize = SHA256_DIGEST_SIZE; + else if (digestsize == SHA384_DIGEST_SIZE) + updated_digestsize = SHA512_DIGEST_SIZE; + if (reqctx->result == 1) { + reqctx->result = 0; + memcpy(req->result, input + sizeof(struct cpl_fw6_pld), + digestsize); + } else { + memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld), + updated_digestsize); + } +out: + req->base.complete(&req->base, err); + + } + +static inline void chcr_handle_aead_resp(struct aead_request *req, + unsigned char *input, + int err) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm)); + + + chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op); + if (reqctx->b0_dma) + dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma, + reqctx->b0_len, DMA_BIDIRECTIONAL); + if (reqctx->verify == VERIFY_SW) { + chcr_verify_tag(req, input, &err); + reqctx->verify = VERIFY_HW; +} + req->base.complete(&req->base, err); + +} static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) { u8 temp[SHA512_DIGEST_SIZE]; @@ -166,27 +239,11 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, { struct crypto_tfm *tfm = req->tfm; struct chcr_context *ctx = crypto_tfm_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_req_ctx ctx_req; - unsigned int digestsize, updated_digestsize; struct adapter *adap = padap(ctx->dev); switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_AEAD: - ctx_req.req.aead_req = aead_request_cast(req); - ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req); - dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst, - ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE); - if (ctx_req.ctx.reqctx->skb) { - kfree_skb(ctx_req.ctx.reqctx->skb); - ctx_req.ctx.reqctx->skb = NULL; - } - if (ctx_req.ctx.reqctx->verify == VERIFY_SW) { - chcr_verify_tag(ctx_req.req.aead_req, input, - &err); - ctx_req.ctx.reqctx->verify = VERIFY_HW; - } - ctx_req.req.aead_req->base.complete(req, err); + chcr_handle_aead_resp(aead_request_cast(req), input, err); break; case CRYPTO_ALG_TYPE_ABLKCIPHER: @@ -195,60 +252,13 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, break; case CRYPTO_ALG_TYPE_AHASH: - ctx_req.req.ahash_req = ahash_request_cast(req); - ctx_req.ctx.ahash_ctx = - ahash_request_ctx(ctx_req.req.ahash_req); - digestsize = - crypto_ahash_digestsize(crypto_ahash_reqtfm( - ctx_req.req.ahash_req)); - updated_digestsize = digestsize; - if (digestsize == SHA224_DIGEST_SIZE) - updated_digestsize = SHA256_DIGEST_SIZE; - else if (digestsize == SHA384_DIGEST_SIZE) - updated_digestsize = SHA512_DIGEST_SIZE; - if (ctx_req.ctx.ahash_ctx->skb) { - kfree_skb(ctx_req.ctx.ahash_ctx->skb); - ctx_req.ctx.ahash_ctx->skb = NULL; - } - if (ctx_req.ctx.ahash_ctx->result == 1) { - ctx_req.ctx.ahash_ctx->result = 0; - memcpy(ctx_req.req.ahash_req->result, input + - sizeof(struct cpl_fw6_pld), - digestsize); - } else { - memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input + - sizeof(struct cpl_fw6_pld), - updated_digestsize); + chcr_handle_ahash_resp(ahash_request_cast(req), input, err); } - ctx_req.req.ahash_req->base.complete(req, err); - break; - } atomic_inc(&adap->chcr_stats.complete); return err; } -/* - * calc_tx_flits_ofld - calculate # of flits for an offload packet - * @skb: the packet - * Returns the number of flits needed for the given offload packet. - * These packets are already fully constructed and no additional headers - * will be added. - */ -static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) -{ - unsigned int flits, cnt; - - if (is_ofld_imm(skb)) - return DIV_ROUND_UP(skb->len, 8); - - flits = skb_transport_offset(skb) / 8; /* headers */ - cnt = skb_shinfo(skb)->nr_frags; - if (skb_tail_pointer(skb) != skb_transport_header(skb)) - cnt++; - return flits + sgl_len(cnt); -} - -static inline void get_aes_decrypt_key(unsigned char *dec_key, +static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength) { @@ -395,64 +405,193 @@ static inline int is_hmac(struct crypto_tfm *tfm) return 0; } -static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl, - struct scatterlist *sg, - struct phys_sge_parm *sg_param) +static inline void dsgl_walk_init(struct dsgl_walk *walk, + struct cpl_rx_phys_dsgl *dsgl) { - struct phys_sge_pairs *to; - unsigned int len = 0, left_size = sg_param->obsize; - unsigned int j = 0; - int offset, ent_len; + walk->dsgl = dsgl; + walk->nents = 0; + walk->to = (struct phys_sge_pairs *)(dsgl + 1); +} + +static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid) +{ + struct cpl_rx_phys_dsgl *phys_cpl; + + phys_cpl = walk->dsgl; phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); - to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl + - sizeof(struct cpl_rx_phys_dsgl)); + phys_cpl->pcirlxorder_to_noofsgentr = + htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | + CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | + CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | + CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | + CPL_RX_PHYS_DSGL_DCAID_V(0) | + CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents)); + phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; + phys_cpl->rss_hdr_int.qid = htons(qid); + phys_cpl->rss_hdr_int.hash_val = 0; +} + +static inline void dsgl_walk_add_page(struct dsgl_walk *walk, + size_t size, + dma_addr_t *addr) +{ + int j; + + if (!size) + return; + j = walk->nents; + walk->to->len[j % 8] = htons(size); + walk->to->addr[j % 8] = cpu_to_be64(*addr); + j++; + if ((j % 8) == 0) + walk->to++; + walk->nents = j; +} + +static void dsgl_walk_add_sg(struct dsgl_walk *walk, + struct scatterlist *sg, + unsigned int slen, + unsigned int skip) +{ + int skip_len = 0; + unsigned int left_size = slen, len = 0; + unsigned int j = walk->nents; + int offset, ent_len; + + if (!slen) + return; + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; + sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; + } + } + while (left_size && sg) { - len = min_t(u32, left_size, sg_dma_len(sg)); + len = min_t(u32, left_size, sg_dma_len(sg) - skip_len); offset = 0; while (len) { - ent_len = min_t(u32, len, CHCR_SG_SIZE); - to->len[j % 8] = htons(ent_len); - to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + - offset); + ent_len = min_t(u32, len, CHCR_DST_SG_SIZE); + walk->to->len[j % 8] = htons(ent_len); + walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + + offset + skip_len); offset += ent_len; len -= ent_len; j++; if ((j % 8) == 0) - to++; + walk->to++; } - left_size -= min(left_size, sg_dma_len(sg)); + walk->last_sg = sg; + walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) - + skip_len) + skip_len; + left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len); + skip_len = 0; sg = sg_next(sg); } - phys_cpl->pcirlxorder_to_noofsgentr = - htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | - CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | - CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | - CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | - CPL_RX_PHYS_DSGL_DCAID_V(0) | - CPL_RX_PHYS_DSGL_NOOFSGENTR_V(j)); - phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; - phys_cpl->rss_hdr_int.qid = htons(sg_param->qid); - phys_cpl->rss_hdr_int.hash_val = 0; + walk->nents = j; +} + +static inline void ulptx_walk_init(struct ulptx_walk *walk, + struct ulptx_sgl *ulp) +{ + walk->sgl = ulp; + walk->nents = 0; + walk->pair_idx = 0; + walk->pair = ulp->sge; + walk->last_sg = NULL; + walk->last_sg_len = 0; +} + +static inline void ulptx_walk_end(struct ulptx_walk *walk) +{ + walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | + ULPTX_NSGE_V(walk->nents)); +} + +static inline void ulptx_walk_add_page(struct ulptx_walk *walk, + size_t size, + dma_addr_t *addr) +{ + if (!size) + return; + + if (walk->nents == 0) { + walk->sgl->len0 = cpu_to_be32(size); + walk->sgl->addr0 = cpu_to_be64(*addr); + } else { + walk->pair->addr[walk->pair_idx] = cpu_to_be64(*addr); + walk->pair->len[walk->pair_idx] = cpu_to_be32(size); + walk->pair_idx = !walk->pair_idx; + if (!walk->pair_idx) + walk->pair++; + } + walk->nents++; } -static inline int map_writesg_phys_cpl(struct device *dev, - struct cpl_rx_phys_dsgl *phys_cpl, +static void ulptx_walk_add_sg(struct ulptx_walk *walk, struct scatterlist *sg, - struct phys_sge_parm *sg_param) + unsigned int len, + unsigned int skip) { - if (!sg || !sg_param->nents) - return -EINVAL; + int small; + int skip_len = 0; + unsigned int sgmin; - sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE); - if (sg_param->nents == 0) { - pr_err("CHCR : DMA mapping failed\n"); - return -EINVAL; + if (!len) + return; + + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; + sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; + } + } + if (walk->nents == 0) { + small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len); + sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); + walk->sgl->len0 = cpu_to_be32(sgmin); + walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len); + walk->nents++; + len -= sgmin; + walk->last_sg = sg; + walk->last_sg_len = sgmin + skip_len; + skip_len += sgmin; + if (sg_dma_len(sg) == skip_len) { + sg = sg_next(sg); + skip_len = 0; + } + } + + while (sg && len) { + small = min(sg_dma_len(sg) - skip_len, len); + sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); + walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin); + walk->pair->addr[walk->pair_idx] = + cpu_to_be64(sg_dma_address(sg) + skip_len); + walk->pair_idx = !walk->pair_idx; + walk->nents++; + if (!walk->pair_idx) + walk->pair++; + len -= sgmin; + skip_len += sgmin; + walk->last_sg = sg; + walk->last_sg_len = skip_len; + if (sg_dma_len(sg) == skip_len) { + sg = sg_next(sg); + skip_len = 0; + } } - write_phys_cpl(phys_cpl, sg, sg_param); - return 0; } static inline int get_aead_subtype(struct crypto_aead *aead) @@ -472,45 +611,6 @@ static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm) return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; } -static inline void write_buffer_to_skb(struct sk_buff *skb, - unsigned int *frags, - char *bfr, - u8 bfr_len) -{ - skb->len += bfr_len; - skb->data_len += bfr_len; - skb->truesize += bfr_len; - get_page(virt_to_page(bfr)); - skb_fill_page_desc(skb, *frags, virt_to_page(bfr), - offset_in_page(bfr), bfr_len); - (*frags)++; -} - - -static inline void -write_sg_to_skb(struct sk_buff *skb, unsigned int *frags, - struct scatterlist *sg, unsigned int count) -{ - struct page *spage; - unsigned int page_len; - - skb->len += count; - skb->data_len += count; - skb->truesize += count; - - while (count > 0) { - if (!sg || (!(sg->length))) - break; - spage = sg_page(sg); - get_page(spage); - page_len = min(sg->length, count); - skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len); - (*frags)++; - count -= page_len; - sg = sg_next(sg); - } -} - static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) { struct adapter *adap = netdev2adap(dev); @@ -546,32 +646,46 @@ static int generate_copy_rrkey(struct ablk_ctx *ablkctx, static int chcr_sg_ent_in_wr(struct scatterlist *src, struct scatterlist *dst, unsigned int minsg, - unsigned int space) + unsigned int space, + unsigned int srcskip, + unsigned int dstskip) { int srclen = 0, dstlen = 0; - int srcsg = minsg, dstsg = 0; + int srcsg = minsg, dstsg = minsg; int offset = 0, less; - while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) && + if (sg_dma_len(src) == srcskip) { + src = sg_next(src); + srcskip = 0; + } + + if (sg_dma_len(dst) == dstskip) { + dst = sg_next(dst); + dstskip = 0; + } + + while (src && dst && space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { - srclen += src->length; + srclen += (sg_dma_len(src) - srcskip); srcsg++; offset = 0; while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { if (srclen <= dstlen) break; - less = min_t(unsigned int, dst->length - offset, - CHCR_SG_SIZE); + less = min_t(unsigned int, sg_dma_len(dst) - offset - + dstskip, CHCR_DST_SG_SIZE); dstlen += less; offset += less; - if (offset == dst->length) { + if (offset == sg_dma_len(dst)) { dst = sg_next(dst); offset = 0; } dstsg++; + dstskip = 0; } src = sg_next(src); + srcskip = 0; } return min(srclen, dstlen); } @@ -601,24 +715,22 @@ static int chcr_cipher_fallback(struct crypto_skcipher *cipher, } static inline void create_wreq(struct chcr_context *ctx, struct chcr_wr *chcr_req, - void *req, struct sk_buff *skb, + struct crypto_async_request *req, + unsigned int imm, int hash_sz, + unsigned int len16, unsigned int sc_len, unsigned int lcb) { struct uld_ctx *u_ctx = ULD_CTX(ctx); int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; - unsigned int immdatalen = 0; - if (is_ofld_imm(skb)) - immdatalen = skb->data_len; chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE; chcr_req->wreq.pld_size_hash_size = htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); chcr_req->wreq.len16_pkd = - htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP( - (calc_tx_flits_ofld(skb) * 8), 16))); + htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16))); chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid, @@ -626,13 +738,12 @@ static inline void create_wreq(struct chcr_context *ctx, chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id, qid); - chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8), - 16) - ((sizeof(chcr_req->wreq)) >> 4))); + chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) - + ((sizeof(chcr_req->wreq)) >> 4))); - chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen); + chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm); chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + - sizeof(chcr_req->key_ctx) + - sc_len + immdatalen); + sizeof(chcr_req->key_ctx) + sc_len); } /** @@ -645,49 +756,52 @@ static inline void create_wreq(struct chcr_context *ctx, static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; + struct ulptx_sgl *ulptx; struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(wrparam->req); - struct phys_sge_parm sg_param; - unsigned int frags = 0, transhdr_len, phys_dsgl; + unsigned int temp = 0, transhdr_len, dst_size; int error; int nents; - unsigned int ivsize = AES_BLOCK_SIZE, kctx_len; + unsigned int kctx_len; gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(c_ctx(tfm)->dev); - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, wrparam->bytes); - nents = dstsg_2k(reqctx->dst, wrparam->bytes); - phys_dsgl = get_space_for_phys_dsgl(nents); + nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE, + reqctx->dst_ofst); + dst_size = get_space_for_phys_dsgl(nents + 1); kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); - transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); + nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes, + CHCR_SRC_SG_SIZE, reqctx->src_ofst); + temp = reqctx->imm ? (DIV_ROUND_UP((IV + wrparam->req->nbytes), 16) + * 16) : (sgl_len(nents + MIN_CIPHER_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); chcr_req = __skb_put_zero(skb, transhdr_len); chcr_req->sec_cpl.op_ivinsrtofst = - FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 1); + FILL_SEC_CPL_OP_IVINSR(c_ctx(tfm)->dev->rx_channel_id, 2, 1); - chcr_req->sec_cpl.pldlen = htonl(ivsize + wrparam->bytes); + chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes); chcr_req->sec_cpl.aadstart_cipherstop_hi = - FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0); + FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0); chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0, ablkctx->ciph_mode, - 0, 0, ivsize >> 1); + 0, 0, IV >> 1); chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0, - 0, 1, phys_dsgl); + 0, 0, dst_size); chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr; if ((reqctx->op == CHCR_DECRYPT_OP) && @@ -712,26 +826,18 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) } } phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = wrparam->bytes; - sg_param.qid = wrparam->qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) - goto map_fail1; + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); + chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam); + chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid); - skb_set_transport_header(skb, transhdr_len); - write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); - write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes); atomic_inc(&adap->chcr_stats.cipher_rqst); - create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, 0, - sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl + kctx_len, + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + +(reqctx->imm ? (IV + wrparam->bytes) : 0); + create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0, + transhdr_len, temp, ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); reqctx->skb = skb; - skb_get(skb); return skb; -map_fail1: - kfree_skb(skb); err: return ERR_PTR(error); } @@ -756,8 +862,7 @@ static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher, unsigned int keylen) { struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); int err = 0; crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK); @@ -775,8 +880,7 @@ static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned int ck_size, context_size; u16 alignment = 0; int err; @@ -808,8 +912,7 @@ static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned int ck_size, context_size; u16 alignment = 0; int err; @@ -840,8 +943,7 @@ static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned int ck_size, context_size; u16 alignment = 0; int err; @@ -908,8 +1010,7 @@ static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes) static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); struct crypto_cipher *cipher; int ret, i; @@ -926,8 +1027,7 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) ret = crypto_cipher_setkey(cipher, key, keylen); if (ret) goto out; - - crypto_cipher_encrypt_one(cipher, iv, iv); + /*H/W sends the encrypted IV in dsgl when AADIVDROP bit is 0*/ for (i = 0; i < round8; i++) gf128mul_x8_ble((le128 *)iv, (le128 *)iv); @@ -1005,64 +1105,60 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, unsigned char *input, int err) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct sk_buff *skb; struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input; struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); struct cipher_wr_param wrparam; int bytes; - dma_unmap_sg(&u_ctx->lldi.pdev->dev, reqctx->dst, reqctx->dst_nents, - DMA_FROM_DEVICE); - - if (reqctx->skb) { - kfree_skb(reqctx->skb); - reqctx->skb = NULL; - } if (err) - goto complete; - + goto unmap; if (req->nbytes == reqctx->processed) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); err = chcr_final_cipher_iv(req, fw6_pld, req->info); goto complete; } if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + c_ctx(tfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { err = -EBUSY; - goto complete; + goto unmap; } } - wrparam.srcsg = scatterwalk_ffwd(reqctx->srcffwd, req->src, - reqctx->processed); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, reqctx->dstsg, - reqctx->processed); - if (!wrparam.srcsg || !reqctx->dst) { - pr_err("Input sg list length less that nbytes\n"); - err = -EINVAL; - goto complete; - } - bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1, - SPACE_LEFT(ablkctx->enckey_len)); + if (!reqctx->imm) { + bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 1, + SPACE_LEFT(ablkctx->enckey_len), + reqctx->src_ofst, reqctx->dst_ofst); if ((bytes + reqctx->processed) >= req->nbytes) bytes = req->nbytes - reqctx->processed; else bytes = ROUND_16(bytes); + } else { + /*CTR mode counter overfloa*/ + bytes = req->nbytes - reqctx->processed; + } + dma_sync_single_for_cpu(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv); + dma_sync_single_for_device(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); if (err) - goto complete; + goto unmap; if (unlikely(bytes == 0)) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); err = chcr_cipher_fallback(ablkctx->sw_cipher, req->base.flags, - wrparam.srcsg, - reqctx->dst, - req->nbytes - reqctx->processed, - reqctx->iv, + req->src, + req->dst, + req->nbytes, + req->info, reqctx->op); goto complete; } @@ -1070,21 +1166,23 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == CRYPTO_ALG_SUB_TYPE_CTR) bytes = adjust_ctr_overflow(reqctx->iv, bytes); - reqctx->processed += bytes; - reqctx->last_req_len = bytes; - wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; + wrparam.qid = u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx]; wrparam.req = req; wrparam.bytes = bytes; skb = create_cipher_wr(&wrparam); if (IS_ERR(skb)) { pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); err = PTR_ERR(skb); - goto complete; + goto unmap; } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); chcr_send_wr(skb); + reqctx->last_req_len = bytes; + reqctx->processed += bytes; return 0; +unmap: + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); complete: req->base.complete(&req->base, err); return err; @@ -1098,8 +1196,7 @@ static int process_cipher(struct ablkcipher_request *req, struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct cipher_wr_param wrparam; int bytes, err = -EINVAL; @@ -1113,16 +1210,41 @@ static int process_cipher(struct ablkcipher_request *req, ablkctx->enckey_len, req->nbytes, ivsize); goto error; } - wrparam.srcsg = req->src; - reqctx->dstsg = req->dst; - bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG, - SPACE_LEFT(ablkctx->enckey_len)); + chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); + if (req->nbytes < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) + + AES_MIN_KEY_SIZE + + sizeof(struct cpl_rx_phys_dsgl) + + /*Min dsgl size*/ + 32))) { + /* Can be sent as Imm*/ + unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len; + + dnents = sg_nents_xlen(req->dst, req->nbytes, + CHCR_DST_SG_SIZE, 0); + dnents += 1; // IV + phys_dsgl = get_space_for_phys_dsgl(dnents); + kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); + transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); + reqctx->imm = (transhdr_len + IV + req->nbytes) <= + SGE_MAX_WR_LEN; + bytes = IV + req->nbytes; + + } else { + reqctx->imm = 0; + } + + if (!reqctx->imm) { + bytes = chcr_sg_ent_in_wr(req->src, req->dst, + MIN_CIPHER_SG, + SPACE_LEFT(ablkctx->enckey_len), + 0, 0); if ((bytes + reqctx->processed) >= req->nbytes) bytes = req->nbytes - reqctx->processed; else bytes = ROUND_16(bytes); - if (unlikely(bytes > req->nbytes)) + } else { bytes = req->nbytes; + } if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == CRYPTO_ALG_SUB_TYPE_CTR) { bytes = adjust_ctr_overflow(req->info, bytes); @@ -1139,9 +1261,11 @@ static int process_cipher(struct ablkcipher_request *req, } else { - memcpy(reqctx->iv, req->info, ivsize); + memcpy(reqctx->iv, req->info, IV); } if (unlikely(bytes == 0)) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); err = chcr_cipher_fallback(ablkctx->sw_cipher, req->base.flags, req->src, @@ -1151,20 +1275,25 @@ static int process_cipher(struct ablkcipher_request *req, op_type); goto error; } - reqctx->processed = bytes; - reqctx->last_req_len = bytes; - reqctx->dst = reqctx->dstsg; reqctx->op = op_type; + reqctx->srcsg = req->src; + reqctx->dstsg = req->dst; + reqctx->src_ofst = 0; + reqctx->dst_ofst = 0; wrparam.qid = qid; wrparam.req = req; wrparam.bytes = bytes; *skb = create_cipher_wr(&wrparam); if (IS_ERR(*skb)) { err = PTR_ERR(*skb); - goto error; + goto unmap; } + reqctx->processed = bytes; + reqctx->last_req_len = bytes; return 0; +unmap: + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); error: return err; } @@ -1172,23 +1301,22 @@ static int process_cipher(struct ablkcipher_request *req, static int chcr_aes_encrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); struct sk_buff *skb = NULL; int err; - struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + c_ctx(tfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } - err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb, - CHCR_ENCRYPT_OP); + err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx], + &skb, CHCR_ENCRYPT_OP); if (err || !skb) return err; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -1196,23 +1324,22 @@ static int chcr_aes_encrypt(struct ablkcipher_request *req) static int chcr_aes_decrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); struct sk_buff *skb = NULL; int err; if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + c_ctx(tfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } - err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb, - CHCR_DECRYPT_OP); + err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx], + &skb, CHCR_DECRYPT_OP); if (err || !skb) return err; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -1360,17 +1487,19 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); + struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); struct sk_buff *skb = NULL; + struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); struct chcr_wr *chcr_req; - unsigned int frags = 0, transhdr_len, iopad_alignment = 0; + struct ulptx_sgl *ulptx; + unsigned int nents = 0, transhdr_len, iopad_alignment = 0; unsigned int digestsize = crypto_ahash_digestsize(tfm); - unsigned int kctx_len = 0; + unsigned int kctx_len = 0, temp = 0; u8 hash_size_in_response = 0; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(h_ctx(tfm)->dev); + int error = 0; iopad_alignment = KEYCTX_ALIGN_PAD(digestsize); kctx_len = param->alg_prm.result_size + iopad_alignment; @@ -1382,15 +1511,22 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, else hash_size_in_response = param->alg_prm.result_size; transhdr_len = HASH_TRANSHDR_SIZE(kctx_len); - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + req_ctx->imm = (transhdr_len + param->bfr_len + param->sg_len) <= + SGE_MAX_WR_LEN; + nents = sg_nents_xlen(req->src, param->sg_len, CHCR_SRC_SG_SIZE, 0); + nents += param->bfr_len ? 1 : 0; + transhdr_len += req_ctx->imm ? (DIV_ROUND_UP((param->bfr_len + + param->sg_len), 16) * 16) : + (sgl_len(nents) * 8); + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) - return skb; - - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); + return ERR_PTR(-ENOMEM); chcr_req = __skb_put_zero(skb, transhdr_len); chcr_req->sec_cpl.op_ivinsrtofst = - FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 0); + FILL_SEC_CPL_OP_IVINSR(h_ctx(tfm)->dev->rx_channel_id, 2, 0); chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len); chcr_req->sec_cpl.aadstart_cipherstop_hi = @@ -1419,37 +1555,52 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, ((kctx_len + sizeof(chcr_req->key_ctx)) >> 4)); chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1); - - skb_set_transport_header(skb, transhdr_len); - if (param->bfr_len != 0) - write_buffer_to_skb(skb, &frags, req_ctx->reqbfr, - param->bfr_len); - if (param->sg_len != 0) - write_sg_to_skb(skb, &frags, req->src, param->sg_len); + ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + kctx_len + + DUMMY_BYTES); + if (param->bfr_len != 0) { + req_ctx->dma_addr = dma_map_single(&u_ctx->lldi.pdev->dev, + req_ctx->reqbfr, param->bfr_len, + DMA_TO_DEVICE); + if (dma_mapping_error(&u_ctx->lldi.pdev->dev, + req_ctx->dma_addr)) { + error = -ENOMEM; + goto err; + } + req_ctx->dma_len = param->bfr_len; + } else { + req_ctx->dma_addr = 0; + } + chcr_add_hash_src_ent(req, ulptx, param); + /* Request upto max wr size */ + temp = kctx_len + DUMMY_BYTES + (req_ctx->imm ? (param->sg_len + + param->bfr_len) : 0); atomic_inc(&adap->chcr_stats.digest_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, hash_size_in_response, - DUMMY_BYTES + kctx_len, 0); + create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->imm, + hash_size_in_response, transhdr_len, + temp, 0); req_ctx->skb = skb; - skb_get(skb); return skb; +err: + kfree_skb(skb); + return ERR_PTR(error); } static int chcr_ahash_update(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct uld_ctx *u_ctx = NULL; struct sk_buff *skb; u8 remainder = 0, bs; unsigned int nbytes = req->nbytes; struct hash_wr_param params; + int error; bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + h_ctx(rtfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -1463,7 +1614,9 @@ static int chcr_ahash_update(struct ahash_request *req) req_ctx->reqlen += nbytes; return 0; } - + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) + return -ENOMEM; params.opad_needed = 0; params.more = 1; params.last = 0; @@ -1474,8 +1627,10 @@ static int chcr_ahash_update(struct ahash_request *req) req_ctx->result = 0; req_ctx->data_len += params.sg_len + params.bfr_len; skb = create_hash_wr(req, ¶ms); - if (IS_ERR(skb)) - return PTR_ERR(skb); + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } if (remainder) { u8 *temp; @@ -1489,10 +1644,13 @@ static int chcr_ahash_update(struct ahash_request *req) } req_ctx->reqlen = remainder; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + return error; } static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1) @@ -1509,13 +1667,12 @@ static int chcr_ahash_final(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct hash_wr_param params; struct sk_buff *skb; struct uld_ctx *u_ctx = NULL; u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (is_hmac(crypto_ahash_tfm(rtfm))) params.opad_needed = 1; else @@ -1542,7 +1699,7 @@ static int chcr_ahash_final(struct ahash_request *req) return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -1551,17 +1708,17 @@ static int chcr_ahash_finup(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct uld_ctx *u_ctx = NULL; struct sk_buff *skb; struct hash_wr_param params; u8 bs; + int error; bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + h_ctx(rtfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -1587,34 +1744,41 @@ static int chcr_ahash_finup(struct ahash_request *req) params.last = 1; params.more = 0; } + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) + return -ENOMEM; skb = create_hash_wr(req, ¶ms); - if (IS_ERR(skb)) - return PTR_ERR(skb); - + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + return error; } static int chcr_ahash_digest(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct uld_ctx *u_ctx = NULL; struct sk_buff *skb; struct hash_wr_param params; u8 bs; + int error; rtfm->init(req); bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + h_ctx(rtfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -1623,6 +1787,9 @@ static int chcr_ahash_digest(struct ahash_request *req) params.opad_needed = 1; else params.opad_needed = 0; + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) + return -ENOMEM; params.last = 0; params.more = 0; @@ -1640,13 +1807,17 @@ static int chcr_ahash_digest(struct ahash_request *req) } skb = create_hash_wr(req, ¶ms); - if (IS_ERR(skb)) - return PTR_ERR(skb); - + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + return error; } static int chcr_ahash_export(struct ahash_request *areq, void *out) @@ -1656,6 +1827,8 @@ static int chcr_ahash_export(struct ahash_request *areq, void *out) state->reqlen = req_ctx->reqlen; state->data_len = req_ctx->data_len; + state->is_sg_map = 0; + state->result = 0; memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen); memcpy(state->partial_hash, req_ctx->partial_hash, CHCR_HASH_MAX_DIGEST_SIZE); @@ -1671,6 +1844,8 @@ static int chcr_ahash_import(struct ahash_request *areq, const void *in) req_ctx->data_len = state->data_len; req_ctx->reqbfr = req_ctx->bfr1; req_ctx->skbfr = req_ctx->bfr2; + req_ctx->is_sg_map = 0; + req_ctx->result = 0; memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128); memcpy(req_ctx->partial_hash, state->partial_hash, CHCR_HASH_MAX_DIGEST_SIZE); @@ -1680,8 +1855,7 @@ static int chcr_ahash_import(struct ahash_request *areq, const void *in) static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); + struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); unsigned int digestsize = crypto_ahash_digestsize(tfm); unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); unsigned int i, err = 0, updated_digestsize; @@ -1734,8 +1908,7 @@ static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int key_len) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned short context_size = 0; int err; @@ -1774,6 +1947,7 @@ static int chcr_sha_init(struct ahash_request *areq) req_ctx->skbfr = req_ctx->bfr2; req_ctx->skb = NULL; req_ctx->result = 0; + req_ctx->is_sg_map = 0; copy_hash_init_values(req_ctx->partial_hash, digestsize); return 0; } @@ -1789,8 +1963,7 @@ static int chcr_hmac_init(struct ahash_request *areq) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); + struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm)); unsigned int digestsize = crypto_ahash_digestsize(rtfm); unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); @@ -1836,29 +2009,48 @@ static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) } } -static int chcr_copy_assoc(struct aead_request *req, - struct chcr_aead_ctx *ctx) +static int chcr_aead_common_init(struct aead_request *req, + unsigned short op_type) { - SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null); - - skcipher_request_set_tfm(skreq, ctx->null); - skcipher_request_set_callback(skreq, aead_request_flags(req), - NULL, NULL); - skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen, - NULL); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + int error = -EINVAL; + unsigned int dst_size; + unsigned int authsize = crypto_aead_authsize(tfm); - return crypto_skcipher_encrypt(skreq); + dst_size = req->assoclen + req->cryptlen + (op_type ? + -authsize : authsize); + /* validate key size */ + if (aeadctx->enckey_len == 0) + goto err; + if (op_type && req->cryptlen < authsize) + goto err; + error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); + if (error) { + error = -ENOMEM; + goto err; + } + reqctx->aad_nents = sg_nents_xlen(req->src, req->assoclen, + CHCR_SRC_SG_SIZE, 0); + reqctx->src_nents = sg_nents_xlen(req->src, req->cryptlen, + CHCR_SRC_SG_SIZE, req->assoclen); + return 0; +err: + return error; } -static int chcr_aead_need_fallback(struct aead_request *req, int src_nent, + +static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents, int aadmax, int wrlen, unsigned short op_type) { unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); if (((req->cryptlen - (op_type ? authsize : 0)) == 0) || + dst_nents > MAX_DSGL_ENT || (req->assoclen > aadmax) || - (src_nent > MAX_SKB_FRAGS) || - (wrlen > MAX_WR_SIZE)) + (wrlen > SGE_MAX_WR_LEN)) return 1; return 0; } @@ -1866,8 +2058,7 @@ static int chcr_aead_need_fallback(struct aead_request *req, int src_nent, static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct aead_request *subreq = aead_request_ctx(req); aead_request_set_tfm(subreq, aeadctx->sw_cipher); @@ -1886,84 +2077,75 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; - struct phys_sge_parm sg_param; - struct scatterlist *src; - unsigned int frags = 0, transhdr_len; - unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0; - unsigned int kctx_len = 0, nents; - unsigned short stop_offset = 0; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len; + unsigned int dst_size = 0, temp; + unsigned int kctx_len = 0, dnents; unsigned int assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); - int error = -EINVAL, src_nent; + int error = -EINVAL; int null = 0; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(a_ctx(tfm)->dev); - dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : - authsize); - if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0)) - goto err; + if (req->cryptlen == 0) + return NULL; - if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) - goto err; - src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); - if (src_nent < 0) - goto err; - src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); - reqctx->dst = src; - if (req->src != req->dst) { - error = chcr_copy_assoc(req, aeadctx); - if (error) - return ERR_PTR(error); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, - req->assoclen); - } + reqctx->b0_dma = 0; if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) { null = 1; assoclen = 0; } - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)); - if (reqctx->dst_nents < 0) { - pr_err("AUTHENC:Invalid Destination sg entries\n"); - error = -EINVAL; - goto err; + dst_size = assoclen + req->cryptlen + (op_type ? -authsize : + authsize); + error = chcr_aead_common_init(req, op_type); + if (error) + return ERR_PTR(error); + if (dst_size) { + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, req->cryptlen + + (op_type ? -authsize : authsize), CHCR_DST_SG_SIZE, + req->assoclen); + dnents += MIN_AUTH_SG; // For IV + } else { + dnents = 0; } - nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)) : 0; - dst_size = get_space_for_phys_dsgl(nents); + + dst_size = get_space_for_phys_dsgl(dnents); kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) - sizeof(chcr_req->key_ctx); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); - if (chcr_aead_need_fallback(req, src_nent + MIN_AUTH_SG, - T6_MAX_AAD_SIZE, - transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8), - op_type)) { + reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) < + SGE_MAX_WR_LEN; + temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen), 16) + * 16) : (sgl_len(reqctx->src_nents + reqctx->aad_nents + + MIN_GCM_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, + transhdr_len, op_type)) { atomic_inc(&adap->chcr_stats.fallback); + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); return ERR_PTR(chcr_aead_fallback(req, op_type)); } - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - /* LLD is going to write the sge hdr. */ - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); - - /* Write WR */ chcr_req = __skb_put_zero(skb, transhdr_len); - stop_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; + temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; /* * Input order is AAD,IV and Payload. where IV should be included as @@ -1971,24 +2153,24 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, * to the hardware spec */ chcr_req->sec_cpl.op_ivinsrtofst = - FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, - (ivsize ? (assoclen + 1) : 0)); - chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen); + FILL_SEC_CPL_OP_IVINSR(a_ctx(tfm)->dev->rx_channel_id, 2, + assoclen + 1); + chcr_req->sec_cpl.pldlen = htonl(assoclen + IV + req->cryptlen); chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( assoclen ? 1 : 0, assoclen, - assoclen + ivsize + 1, - (stop_offset & 0x1F0) >> 4); + assoclen + IV + 1, + (temp & 0x1F0) >> 4); chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT( - stop_offset & 0xF, - null ? 0 : assoclen + ivsize + 1, - stop_offset, stop_offset); + temp & 0xF, + null ? 0 : assoclen + IV + 1, + temp, temp); chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, CHCR_SCMD_CIPHER_MODE_AES_CBC, actx->auth_mode, aeadctx->hmac_ctrl, - ivsize >> 1); + IV >> 1); chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, - 0, 1, dst_size); + 0, 0, dst_size); chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; if (op_type == CHCR_ENCRYPT_OP) @@ -2001,39 +2183,312 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4), actx->h_iopad, kctx_len - (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4)); - + memcpy(reqctx->iv, req->iv, IV); phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); - sg_param.qid = qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) - goto dstmap_fail; - - skb_set_transport_header(skb, transhdr_len); - - if (assoclen) { - /* AAD buffer in */ - write_sg_to_skb(skb, &frags, req->src, assoclen); - - } - write_buffer_to_skb(skb, &frags, req->iv, ivsize); - write_sg_to_skb(skb, &frags, src, req->cryptlen); + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); + chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); + chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); atomic_inc(&adap->chcr_stats.cipher_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, size, - sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + + kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, + transhdr_len, temp, 0); reqctx->skb = skb; - skb_get(skb); + reqctx->op = op_type; return skb; -dstmap_fail: - /* ivmap_fail: */ - kfree_skb(skb); err: + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); + return ERR_PTR(error); } +static int chcr_aead_dma_map(struct device *dev, + struct aead_request *req, + unsigned short op_type) +{ + int error; + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(tfm); + int dst_size; + + dst_size = req->assoclen + req->cryptlen + (op_type ? + -authsize : authsize); + if (!req->cryptlen || !dst_size) + return 0; + reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, reqctx->iv_dma)) + return -ENOMEM; + + if (req->src == req->dst) { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + if (!error) + goto err; + } else { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + if (!error) + goto err; + error = dma_map_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + if (!error) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + goto err; + } + } + + return 0; +err: + dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); + return -ENOMEM; +} + +static void chcr_aead_dma_unmap(struct device *dev, + struct aead_request *req, + unsigned short op_type) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(tfm); + int dst_size; + + dst_size = req->assoclen + req->cryptlen + (op_type ? + -authsize : authsize); + if (!req->cryptlen || !dst_size) + return; + + dma_unmap_single(dev, reqctx->iv_dma, IV, + DMA_BIDIRECTIONAL); + if (req->src == req->dst) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + } +} + +static inline void chcr_add_aead_src_ent(struct aead_request *req, + struct ulptx_sgl *ulptx, + unsigned int assoclen, + unsigned short op_type) +{ + struct ulptx_walk ulp_walk; + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + + if (reqctx->imm) { + u8 *buf = (u8 *)ulptx; + + if (reqctx->b0_dma) { + memcpy(buf, reqctx->scratch_pad, reqctx->b0_len); + buf += reqctx->b0_len; + } + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, assoclen, 0); + buf += assoclen; + memcpy(buf, reqctx->iv, IV); + buf += IV; + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, req->cryptlen, req->assoclen); + } else { + ulptx_walk_init(&ulp_walk, ulptx); + if (reqctx->b0_dma) + ulptx_walk_add_page(&ulp_walk, reqctx->b0_len, + &reqctx->b0_dma); + ulptx_walk_add_sg(&ulp_walk, req->src, assoclen, 0); + ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma); + ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen, + req->assoclen); + ulptx_walk_end(&ulp_walk); + } +} + +static inline void chcr_add_aead_dst_ent(struct aead_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + unsigned int assoclen, + unsigned short op_type, + unsigned short qid) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct dsgl_walk dsgl_walk; + unsigned int authsize = crypto_aead_authsize(tfm); + u32 temp; + + dsgl_walk_init(&dsgl_walk, phys_cpl); + if (reqctx->b0_dma) + dsgl_walk_add_page(&dsgl_walk, reqctx->b0_len, &reqctx->b0_dma); + dsgl_walk_add_sg(&dsgl_walk, req->dst, assoclen, 0); + dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma); + temp = req->cryptlen + (op_type ? -authsize : authsize); + dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, req->assoclen); + dsgl_walk_end(&dsgl_walk, qid); +} + +static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, + struct ulptx_sgl *ulptx, + struct cipher_wr_param *wrparam) +{ + struct ulptx_walk ulp_walk; + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + + if (reqctx->imm) { + u8 *buf = (u8 *)ulptx; + + memcpy(buf, reqctx->iv, IV); + buf += IV; + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, wrparam->bytes, reqctx->processed); + } else { + ulptx_walk_init(&ulp_walk, ulptx); + ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma); + ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes, + reqctx->src_ofst); + reqctx->srcsg = ulp_walk.last_sg; + reqctx->src_ofst = ulp_walk.last_sg_len; + ulptx_walk_end(&ulp_walk); + } +} + +static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + struct cipher_wr_param *wrparam, + unsigned short qid) +{ + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + struct dsgl_walk dsgl_walk; + + dsgl_walk_init(&dsgl_walk, phys_cpl); + dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma); + dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes, + reqctx->dst_ofst); + reqctx->dstsg = dsgl_walk.last_sg; + reqctx->dst_ofst = dsgl_walk.last_sg_len; + + dsgl_walk_end(&dsgl_walk, qid); +} + +static inline void chcr_add_hash_src_ent(struct ahash_request *req, + struct ulptx_sgl *ulptx, + struct hash_wr_param *param) +{ + struct ulptx_walk ulp_walk; + struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); + + if (reqctx->imm) { + u8 *buf = (u8 *)ulptx; + + if (param->bfr_len) { + memcpy(buf, reqctx->reqbfr, param->bfr_len); + buf += param->bfr_len; + } + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, param->sg_len, 0); + } else { + ulptx_walk_init(&ulp_walk, ulptx); + if (param->bfr_len) + ulptx_walk_add_page(&ulp_walk, param->bfr_len, + &reqctx->dma_addr); + ulptx_walk_add_sg(&ulp_walk, req->src, param->sg_len, + 0); +// reqctx->srcsg = ulp_walk.last_sg; +// reqctx->src_ofst = ulp_walk.last_sg_len; + ulptx_walk_end(&ulp_walk); + } +} + + +static inline int chcr_hash_dma_map(struct device *dev, + struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + int error = 0; + + if (!req->nbytes) + return 0; + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + if (!error) + return error; + req_ctx->is_sg_map = 1; + return 0; +} + +static inline void chcr_hash_dma_unmap(struct device *dev, + struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + + if (!req->nbytes) + return; + + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + req_ctx->is_sg_map = 0; + +} + + +static int chcr_cipher_dma_map(struct device *dev, + struct ablkcipher_request *req) +{ + int error; + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + + reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, reqctx->iv_dma)) + return -ENOMEM; + + if (req->src == req->dst) { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + if (!error) + goto err; + } else { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + if (!error) + goto err; + error = dma_map_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + if (!error) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + goto err; + } + } + + return 0; +err: + dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); + return -ENOMEM; +} +static void chcr_cipher_dma_unmap(struct device *dev, + struct ablkcipher_request *req) +{ + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + + dma_unmap_single(dev, reqctx->iv_dma, IV, + DMA_BIDIRECTIONAL); + if (req->src == req->dst) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + } +} + static int set_msg_len(u8 *block, unsigned int msglen, int csize) { __be32 data; @@ -2118,15 +2573,13 @@ static int ccm_format_packet(struct aead_request *req, static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, unsigned int dst_size, struct aead_request *req, - unsigned short op_type, - struct chcr_context *chcrctx) + unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); - unsigned int ivsize = AES_BLOCK_SIZE; + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; - unsigned int c_id = chcrctx->dev->rx_channel_id; + unsigned int c_id = a_ctx(tfm)->dev->rx_channel_id; unsigned int ccm_xtra; unsigned char tag_offset = 0, auth_offset = 0; unsigned int assoclen; @@ -2139,7 +2592,7 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, ((assoclen) ? CCM_AAD_FIELD_SIZE : 0); auth_offset = req->cryptlen ? - (assoclen + ivsize + 1 + ccm_xtra) : 0; + (assoclen + IV + 1 + ccm_xtra) : 0; if (op_type == CHCR_DECRYPT_OP) { if (crypto_aead_authsize(tfm) != req->cryptlen) tag_offset = crypto_aead_authsize(tfm); @@ -2149,14 +2602,13 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id, - 2, (ivsize ? (assoclen + 1) : 0) + - ccm_xtra); + 2, assoclen + 1 + ccm_xtra); sec_cpl->pldlen = - htonl(assoclen + ivsize + req->cryptlen + ccm_xtra); + htonl(assoclen + IV + req->cryptlen + ccm_xtra); /* For CCM there wil be b0 always. So AAD start will be 1 always */ sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 1, assoclen + ccm_xtra, assoclen - + ivsize + 1 + ccm_xtra, 0); + + IV + 1 + ccm_xtra, 0); sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, auth_offset, tag_offset, @@ -2165,10 +2617,10 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 0 : 1, cipher_mode, mac_mode, - aeadctx->hmac_ctrl, ivsize >> 1); + aeadctx->hmac_ctrl, IV >> 1); sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, - 1, dst_size); + 0, dst_size); } int aead_ccm_validate_input(unsigned short op_type, @@ -2188,119 +2640,83 @@ int aead_ccm_validate_input(unsigned short op_type, return -EINVAL; } } - if (aeadctx->enckey_len == 0) { - pr_err("CCM: Encryption key not set\n"); - return -EINVAL; - } return 0; } -unsigned int fill_aead_req_fields(struct sk_buff *skb, - struct aead_request *req, - struct scatterlist *src, - unsigned int ivsize, - struct chcr_aead_ctx *aeadctx) -{ - unsigned int frags = 0; - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); - /* b0 and aad length(if available) */ - - write_buffer_to_skb(skb, &frags, reqctx->scratch_pad, CCM_B0_SIZE + - (req->assoclen ? CCM_AAD_FIELD_SIZE : 0)); - if (req->assoclen) { - if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) - write_sg_to_skb(skb, &frags, req->src, - req->assoclen - 8); - else - write_sg_to_skb(skb, &frags, req->src, req->assoclen); - } - write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); - if (req->cryptlen) - write_sg_to_skb(skb, &frags, src, req->cryptlen); - - return frags; -} - static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, unsigned short qid, int size, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; - struct phys_sge_parm sg_param; - struct scatterlist *src; - unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE; - unsigned int dst_size = 0, kctx_len, nents; - unsigned int sub_type; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len; + unsigned int dst_size = 0, kctx_len, dnents, temp; + unsigned int sub_type, assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); - int error = -EINVAL, src_nent; + int error = -EINVAL; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(a_ctx(tfm)->dev); - dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : + reqctx->b0_dma = 0; + sub_type = get_aead_subtype(tfm); + if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) + assoclen -= 8; + dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize); - if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) - goto err; - src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); - if (src_nent < 0) - goto err; + error = chcr_aead_common_init(req, op_type); + if (error) + return ERR_PTR(error); - sub_type = get_aead_subtype(tfm); - src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); - reqctx->dst = src; - if (req->src != req->dst) { - error = chcr_copy_assoc(req, aeadctx); - if (error) { - pr_err("AAD copy to destination buffer fails\n"); - return ERR_PTR(error); - } - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, - req->assoclen); - } - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)); - if (reqctx->dst_nents < 0) { - pr_err("CCM:Invalid Destination sg entries\n"); - error = -EINVAL; - goto err; - } + + reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0); error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type); if (error) goto err; - nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)) : 0; - dst_size = get_space_for_phys_dsgl(nents); + if (dst_size) { + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, req->cryptlen + + (op_type ? -authsize : authsize), + CHCR_DST_SG_SIZE, req->assoclen); + dnents += MIN_CCM_SG; // For IV and B0 + } else { + dnents = 0; + } + dst_size = get_space_for_phys_dsgl(dnents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); - if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG, - T6_MAX_AAD_SIZE - 18, - transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8), - op_type)) { + reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen + + reqctx->b0_len) <= SGE_MAX_WR_LEN; + temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen + + reqctx->b0_len), 16) * 16) : + (sgl_len(reqctx->src_nents + reqctx->aad_nents + + MIN_CCM_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE - + reqctx->b0_len, transhdr_len, op_type)) { atomic_inc(&adap->chcr_stats.fallback); + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); return ERR_PTR(chcr_aead_fallback(req, op_type)); } - - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); + chcr_req = (struct chcr_wr *) __skb_put_zero(skb, transhdr_len); - chcr_req = __skb_put_zero(skb, transhdr_len); - - fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type, ctx); + fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type); chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); @@ -2308,29 +2724,37 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, 16), aeadctx->key, aeadctx->enckey_len); phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); error = ccm_format_packet(req, aeadctx, sub_type, op_type); if (error) goto dstmap_fail; - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); - sg_param.qid = qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) + reqctx->b0_dma = dma_map_single(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, + &reqctx->scratch_pad, reqctx->b0_len, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, + reqctx->b0_dma)) { + error = -ENOMEM; goto dstmap_fail; + } + + chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); + chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); - skb_set_transport_header(skb, transhdr_len); - frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx); atomic_inc(&adap->chcr_stats.aead_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, 0, - sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + + kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen + + reqctx->b0_len) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0, + transhdr_len, temp, 0); reqctx->skb = skb; - skb_get(skb); + reqctx->op = op_type; + return skb; dstmap_fail: kfree_skb(skb); err: + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type); return ERR_PTR(error); } @@ -2340,101 +2764,84 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; - struct phys_sge_parm sg_param; - struct scatterlist *src; - unsigned int frags = 0, transhdr_len; - unsigned int ivsize = AES_BLOCK_SIZE; - unsigned int dst_size = 0, kctx_len, nents, assoclen = req->assoclen; - unsigned char tag_offset = 0; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len, dnents = 0; + unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); - int error = -EINVAL, src_nent; + int error = -EINVAL; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); - - dst_size = assoclen + req->cryptlen + (op_type ? -authsize : - authsize); - /* validate key size */ - if (aeadctx->enckey_len == 0) - goto err; + struct adapter *adap = padap(a_ctx(tfm)->dev); - if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) - goto err; - src_nent = sg_nents_for_len(req->src, assoclen + req->cryptlen); - if (src_nent < 0) - goto err; + if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) + assoclen = req->assoclen - 8; - src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen); - reqctx->dst = src; - if (req->src != req->dst) { - error = chcr_copy_assoc(req, aeadctx); + reqctx->b0_dma = 0; + dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize); + error = chcr_aead_common_init(req, op_type); if (error) return ERR_PTR(error); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, - req->assoclen); - } - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)); - if (reqctx->dst_nents < 0) { - pr_err("GCM:Invalid Destination sg entries\n"); - error = -EINVAL; - goto err; + if (dst_size) { + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, + req->cryptlen + (op_type ? -authsize : authsize), + CHCR_DST_SG_SIZE, req->assoclen); + dnents += MIN_GCM_SG; // For IV + } else { + dnents = 0; } - - nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)) : 0; - dst_size = get_space_for_phys_dsgl(nents); + dst_size = get_space_for_phys_dsgl(dnents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) + AEAD_H_SIZE; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); - if (chcr_aead_need_fallback(req, src_nent + MIN_GCM_SG, - T6_MAX_AAD_SIZE, - transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8), - op_type)) { + reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) <= + SGE_MAX_WR_LEN; + temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + + req->cryptlen), 16) * 16) : (sgl_len(reqctx->src_nents + + reqctx->aad_nents + MIN_GCM_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, + transhdr_len, op_type)) { atomic_inc(&adap->chcr_stats.fallback); + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); return ERR_PTR(chcr_aead_fallback(req, op_type)); } - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - /* NIC driver is going to write the sge hdr. */ - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); - chcr_req = __skb_put_zero(skb, transhdr_len); - if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) - assoclen = req->assoclen - 8; - - tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; + //Offset of tag from end + temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( - ctx->dev->rx_channel_id, 2, (ivsize ? - (assoclen + 1) : 0)); + a_ctx(tfm)->dev->rx_channel_id, 2, + (assoclen + 1)); chcr_req->sec_cpl.pldlen = - htonl(assoclen + ivsize + req->cryptlen); + htonl(assoclen + IV + req->cryptlen); chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( assoclen ? 1 : 0, assoclen, - assoclen + ivsize + 1, 0); + assoclen + IV + 1, 0); chcr_req->sec_cpl.cipherstop_lo_authinsert = - FILL_SEC_CPL_AUTHINSERT(0, assoclen + ivsize + 1, - tag_offset, tag_offset); + FILL_SEC_CPL_AUTHINSERT(0, assoclen + IV + 1, + temp, temp); chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, CHCR_SCMD_CIPHER_MODE_AES_GCM, CHCR_SCMD_AUTH_MODE_GHASH, - aeadctx->hmac_ctrl, ivsize >> 1); + aeadctx->hmac_ctrl, IV >> 1); chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, - 0, 1, dst_size); + 0, 0, dst_size); chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) * @@ -2452,30 +2859,21 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01); phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); - sg_param.qid = qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) - goto dstmap_fail; + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); - skb_set_transport_header(skb, transhdr_len); - write_sg_to_skb(skb, &frags, req->src, assoclen); - write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); - write_sg_to_skb(skb, &frags, src, req->cryptlen); + chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); + chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); atomic_inc(&adap->chcr_stats.aead_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, size, - sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, - reqctx->verify); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + + kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, + transhdr_len, temp, reqctx->verify); reqctx->skb = skb; - skb_get(skb); + reqctx->op = op_type; return skb; -dstmap_fail: - /* ivmap_fail: */ - kfree_skb(skb); err: + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type); return ERR_PTR(error); } @@ -2483,8 +2881,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, static int chcr_aead_cra_init(struct crypto_aead *tfm) { - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct aead_alg *alg = crypto_aead_alg(tfm); aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, @@ -2495,25 +2892,20 @@ static int chcr_aead_cra_init(struct crypto_aead *tfm) crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx), sizeof(struct aead_request) + crypto_aead_reqsize(aeadctx->sw_cipher))); - aeadctx->null = crypto_get_default_null_skcipher(); - if (IS_ERR(aeadctx->null)) - return PTR_ERR(aeadctx->null); - return chcr_device_init(ctx); + return chcr_device_init(a_ctx(tfm)); } static void chcr_aead_cra_exit(struct crypto_aead *tfm) { - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); - crypto_put_default_null_skcipher(); crypto_free_aead(aeadctx->sw_cipher); } static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; aeadctx->mayverify = VERIFY_HW; @@ -2522,7 +2914,7 @@ static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, static int chcr_authenc_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); u32 maxauth = crypto_aead_maxauthsize(tfm); /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not @@ -2560,7 +2952,7 @@ static int chcr_authenc_setauthsize(struct crypto_aead *tfm, static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); switch (authsize) { case ICV_4: @@ -2600,7 +2992,7 @@ static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); switch (authsize) { case ICV_8: @@ -2626,7 +3018,7 @@ static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, static int chcr_ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); switch (authsize) { case ICV_4: @@ -2669,8 +3061,7 @@ static int chcr_ccm_common_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); unsigned char ck_size, mk_size; int key_ctx_size = 0; @@ -2703,8 +3094,7 @@ static int chcr_aead_ccm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); int error; crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); @@ -2722,8 +3112,7 @@ static int chcr_aead_ccm_setkey(struct crypto_aead *aead, static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); int error; if (keylen < 3) { @@ -2749,8 +3138,7 @@ static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx); struct crypto_cipher *cipher; unsigned int ck_size; @@ -2822,8 +3210,7 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(authenc); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); /* it contains auth and cipher key both*/ struct crypto_authenc_keys keys; @@ -2943,8 +3330,7 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(authenc); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); struct crypto_authenc_keys keys; int err; @@ -3016,7 +3402,7 @@ static int chcr_aead_encrypt(struct aead_request *req) static int chcr_aead_decrypt(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); int size; @@ -3049,30 +3435,29 @@ static int chcr_aead_op(struct aead_request *req, create_wr_t create_wr_fn) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); struct uld_ctx *u_ctx; struct sk_buff *skb; - if (!ctx->dev) { + if (!a_ctx(tfm)->dev) { pr_err("chcr : %s : No crypto device.\n", __func__); return -ENXIO; } - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(a_ctx(tfm)); if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx)) { + a_ctx(tfm)->tx_qidx)) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } /* Form a WR from req */ - skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], size, + skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size, op_type); if (IS_ERR(skb) || !skb) return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index d1a787b..96c9335 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -214,27 +214,22 @@ calc_tx_flits_ofld(skb) * 8), 16))) #define FILL_CMD_MORE(immdatalen) htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) |\ - ULP_TX_SC_MORE_V((immdatalen) ? 0 : 1)) - + ULP_TX_SC_MORE_V((immdatalen))) #define MAX_NK 8 -#define CRYPTO_MAX_IMM_TX_PKT_LEN 256 -#define MAX_WR_SIZE 512 #define ROUND_16(bytes) ((bytes) & 0xFFFFFFF0) #define MAX_DSGL_ENT 32 -#define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 1) #define MIN_CIPHER_SG 1 /* IV */ -#define MIN_AUTH_SG 2 /*IV + AAD*/ -#define MIN_GCM_SG 2 /* IV + AAD*/ +#define MIN_AUTH_SG 1 /* IV */ +#define MIN_GCM_SG 1 /* IV */ #define MIN_DIGEST_SG 1 /*Partial Buffer*/ -#define MIN_CCM_SG 3 /*IV+AAD+B0*/ +#define MIN_CCM_SG 2 /*IV+B0*/ #define SPACE_LEFT(len) \ - ((MAX_WR_SIZE - WR_MIN_LEN - (len))) + ((SGE_MAX_WR_LEN - WR_MIN_LEN - (len))) -unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, - 48, 64, 72, 88, - 96, 112, 120, 136, - 144, 160, 168, 184, - 192}; +unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88, + 96, 112, 120, 136, 144, 160, 168, 184, + 192, 208, 216, 232, 240, 256, 264, 280, + 288, 304, 312, 328, 336, 352, 360, 376}; unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80, 112, 112, 128, 128, 144, 144, 160, 160, 192, 192, 208, 208, 224, 224, 240, 240, @@ -258,7 +253,6 @@ struct hash_wr_param { struct cipher_wr_param { struct ablkcipher_request *req; - struct scatterlist *srcsg; char *iv; int bytes; unsigned short qid; @@ -298,31 +292,11 @@ enum { ICV_16 = 16 }; -struct hash_op_params { - unsigned char mk_size; - unsigned char pad_align; - unsigned char auth_mode; - char hash_name[MAX_HASH_NAME]; - unsigned short block_size; - unsigned short word_size; - unsigned short ipad_size; -}; - struct phys_sge_pairs { __be16 len[8]; __be64 addr[8]; }; -struct phys_sge_parm { - unsigned int nents; - unsigned int obsize; - unsigned short qid; -}; - -struct crypto_result { - struct completion completion; - int err; -}; static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 8f436f8..94a87e3 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -149,9 +149,23 @@ #define CHCR_HASH_MAX_BLOCK_SIZE_64 64 #define CHCR_HASH_MAX_BLOCK_SIZE_128 128 -#define CHCR_SG_SIZE 2048 +#define CHCR_SRC_SG_SIZE (0x10000 - sizeof(int)) +#define CHCR_DST_SG_SIZE 2048 -/* Aligned to 128 bit boundary */ +static inline struct chcr_context *a_ctx(struct crypto_aead *tfm) +{ + return crypto_aead_ctx(tfm); +} + +static inline struct chcr_context *c_ctx(struct crypto_ablkcipher *tfm) +{ + return crypto_ablkcipher_ctx(tfm); +} + +static inline struct chcr_context *h_ctx(struct crypto_ahash *tfm) +{ + return crypto_tfm_ctx(crypto_ahash_tfm(tfm)); +} struct ablk_ctx { struct crypto_skcipher *sw_cipher; @@ -165,15 +179,39 @@ struct ablk_ctx { }; struct chcr_aead_reqctx { struct sk_buff *skb; - struct scatterlist *dst; - struct scatterlist srcffwd[2]; - struct scatterlist dstffwd[2]; + dma_addr_t iv_dma; + dma_addr_t b0_dma; + unsigned int b0_len; + unsigned int op; + short int aad_nents; + short int src_nents; short int dst_nents; + u16 imm; u16 verify; u8 iv[CHCR_MAX_CRYPTO_IV_LEN]; unsigned char scratch_pad[MAX_SCRATCH_PAD_SIZE]; }; +struct ulptx_walk { + struct ulptx_sgl *sgl; + unsigned int nents; + unsigned int pair_idx; + unsigned int last_sg_len; + struct scatterlist *last_sg; + struct ulptx_sge_pair *pair; + +}; + +struct dsgl_walk { + unsigned int nents; + unsigned int last_sg_len; + struct scatterlist *last_sg; + struct cpl_rx_phys_dsgl *dsgl; + struct phys_sge_pairs *to; +}; + + + struct chcr_gcm_ctx { u8 ghash_h[AEAD_H_SIZE]; }; @@ -194,7 +232,6 @@ struct __aead_ctx { struct chcr_aead_ctx { __be32 key_ctx_hdr; unsigned int enckey_len; - struct crypto_skcipher *null; struct crypto_aead *sw_cipher; u8 salt[MAX_SALT]; u8 key[CHCR_AES_MAX_KEY_LEN]; @@ -230,8 +267,11 @@ struct chcr_ahash_req_ctx { u8 bfr2[CHCR_HASH_MAX_BLOCK_SIZE_128]; u8 *reqbfr; u8 *skbfr; + dma_addr_t dma_addr; + u32 dma_len; u8 reqlen; - /* DMA the partial hash in it */ + u8 imm; + u8 is_sg_map; u8 partial_hash[CHCR_HASH_MAX_DIGEST_SIZE]; u64 data_len; /* Data len till time */ /* SKB which is being sent to the hardware for processing */ @@ -240,14 +280,15 @@ struct chcr_ahash_req_ctx { struct chcr_blkcipher_req_ctx { struct sk_buff *skb; - struct scatterlist srcffwd[2]; - struct scatterlist dstffwd[2]; struct scatterlist *dstsg; - struct scatterlist *dst; unsigned int processed; unsigned int last_req_len; + struct scatterlist *srcsg; + unsigned int src_ofst; + unsigned int dst_ofst; unsigned int op; - short int dst_nents; + dma_addr_t iv_dma; + u16 imm; u8 iv[CHCR_MAX_CRYPTO_IV_LEN]; }; @@ -261,24 +302,6 @@ struct chcr_alg_template { } alg; }; -struct chcr_req_ctx { - union { - struct ahash_request *ahash_req; - struct aead_request *aead_req; - struct ablkcipher_request *ablk_req; - } req; - union { - struct chcr_ahash_req_ctx *ahash_ctx; - struct chcr_aead_reqctx *reqctx; - struct chcr_blkcipher_req_ctx *ablk_ctx; - } ctx; -}; - -struct sge_opaque_hdr { - void *dev; - dma_addr_t addr[MAX_SKB_FRAGS + 1]; -}; - typedef struct sk_buff *(*create_wr_t)(struct aead_request *req, unsigned short qid, int size, @@ -291,4 +314,37 @@ static int chcr_aead_op(struct aead_request *req_base, static inline int get_aead_subtype(struct crypto_aead *aead); static int chcr_handle_cipher_resp(struct ablkcipher_request *req, unsigned char *input, int err); +static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err); +static int chcr_aead_dma_map(struct device *dev, struct aead_request *req, + unsigned short op_type); +static void chcr_aead_dma_unmap(struct device *dev, struct aead_request + *req, unsigned short op_type); +static inline void chcr_add_aead_dst_ent(struct aead_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + unsigned int assoclen, + unsigned short op_type, + unsigned short qid); +static inline void chcr_add_aead_src_ent(struct aead_request *req, + struct ulptx_sgl *ulptx, + unsigned int assoclen, + unsigned short op_type); +static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, + struct ulptx_sgl *ulptx, + struct cipher_wr_param *wrparam); +static int chcr_cipher_dma_map(struct device *dev, + struct ablkcipher_request *req); +static void chcr_cipher_dma_unmap(struct device *dev, + struct ablkcipher_request *req); +static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + struct cipher_wr_param *wrparam, + unsigned short qid); +int sg_nents_len_skip(struct scatterlist *sg, u64 len, u64 skip); +static inline void chcr_add_hash_src_ent(struct ahash_request *req, + struct ulptx_sgl *ulptx, + struct hash_wr_param *param); +static inline int chcr_hash_dma_map(struct device *dev, + struct ahash_request *req); +static inline void chcr_hash_dma_unmap(struct device *dev, + struct ahash_request *req); #endif /* __CHCR_CRYPTO_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c index ede1220..1c1cc48 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -1537,7 +1537,13 @@ int t4_mgmt_tx(struct adapter *adap, struct sk_buff *skb) */ static inline int is_ofld_imm(const struct sk_buff *skb) { - return skb->len <= MAX_IMM_TX_PKT_LEN; + struct work_request_hdr *req = (struct work_request_hdr *)skb->data; + unsigned long opcode = FW_WR_OP_G(ntohl(req->wr_hi)); + + if (opcode == FW_CRYPTO_LOOKASIDE_WR) + return skb->len <= SGE_MAX_WR_LEN; + else + return skb->len <= MAX_IMM_TX_PKT_LEN; } /** -- 2.1.4 ^ permalink raw reply related [flat|nested] 18+ messages in thread
* [PATCH v2 7/7] crypto:chelsio: Fix memory leak [not found] ` <cover.1507449374.git.harsh@chelsio.com> ` (5 preceding siblings ...) 2017-10-08 8:07 ` [PATCH v2 6/7] crypto:chelsio:Move DMA un/mapping to chcr from lld cxgb4 driver Harsh Jain @ 2017-10-08 8:07 ` Harsh Jain 6 siblings, 0 replies; 18+ messages in thread From: Harsh Jain @ 2017-10-08 8:07 UTC (permalink / raw) To: herbert, linux-crypto, netdev; +Cc: Harsh Jain Fix memory leak when device does not support crypto. Reported-by: Dan Carpenter <dan.carpenter@oracle.com> Signed-off-by: Harsh Jain <harsh@chelsio.com> --- drivers/crypto/chelsio/chcr_core.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index b6dd9cb..4f677b3 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -154,15 +154,15 @@ static void *chcr_uld_add(const struct cxgb4_lld_info *lld) struct uld_ctx *u_ctx; /* Create the device and add it in the device list */ + if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE)) + return ERR_PTR(-EOPNOTSUPP); + + /* Create the device and add it in the device list */ u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL); if (!u_ctx) { u_ctx = ERR_PTR(-ENOMEM); goto out; } - if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE)) { - u_ctx = ERR_PTR(-ENOMEM); - goto out; - } u_ctx->lldi = *lld; out: return u_ctx; -- 2.1.4 ^ permalink raw reply related [flat|nested] 18+ messages in thread
end of thread, other threads:[~2017-11-03 14:21 UTC | newest]
Thread overview: 18+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2017-10-03 6:46 [PATCH 0/7]crypto:chelsio: Bugs fixes Harsh Jain
2017-10-03 6:46 ` [PATCH 1/7] crypto:chelsio: Remove unused parameter Harsh Jain
2017-10-03 6:46 ` [PATCH 2/7] crypto:chelsio: Check error code with IS_ERR macro Harsh Jain
2017-10-03 6:46 ` [PATCH 3/7] crypto:gf128mul: The x8_ble multiplication functions Harsh Jain
2017-10-03 14:58 ` David Laight
2017-10-05 11:20 ` Harsh Jain
2017-10-03 6:46 ` [PATCH 4/7] crypto:chelsio:Use x8_ble gf multiplication to calculate IV Harsh Jain
2017-10-03 6:46 ` [PATCH 5/7] crypto:chelsio:Remove allocation of sg list to implement 2K limit of dsgl header Harsh Jain
2017-10-03 6:46 ` [PATCH 6/7] crypto:chelsio:Move DMA un/mapping to chcr from lld cxgb4 driver Harsh Jain
2017-10-03 6:46 ` [PATCH 7/7] crypto:chelsio: Fix memory leak Harsh Jain
[not found] ` <cover.1507449374.git.harsh@chelsio.com>
2017-10-08 8:07 ` [PATCH v2 1/7] crypto:chelsio: Remove unused parameter Harsh Jain
2017-11-03 14:21 ` Herbert Xu
2017-10-08 8:07 ` [PATCH v2 2/7] crypto:chelsio: Check error code with IS_ERR macro Harsh Jain
2017-10-08 8:07 ` [PATCH v2 3/7] crypto:gf128mul: The x8_ble multiplication functions Harsh Jain
2017-10-08 8:07 ` [PATCH v2 4/7] crypto:chelsio:Use x8_ble gf multiplication to calculate IV Harsh Jain
2017-10-08 8:07 ` [PATCH v2 5/7] crypto:chelsio:Remove allocation of sg list to implement 2K limit of dsgl header Harsh Jain
2017-10-08 8:07 ` [PATCH v2 6/7] crypto:chelsio:Move DMA un/mapping to chcr from lld cxgb4 driver Harsh Jain
2017-10-08 8:07 ` [PATCH v2 7/7] crypto:chelsio: Fix memory leak Harsh Jain
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.