From mboxrd@z Thu Jan 1 00:00:00 1970 From: Anoob Joseph Subject: [PATCH 3/3] crypto/octeontx: move device specific code to driver Date: Fri, 1 Mar 2019 18:42:32 +0000 Message-ID: <1551465683-26721-3-git-send-email-anoobj@marvell.com> References: <1551465683-26721-1-git-send-email-anoobj@marvell.com> Mime-Version: 1.0 Content-Type: text/plain; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable Cc: Anoob Joseph , Jerin Jacob Kollanukkaran , Narayana Prasad Raju Athreya , Suheil Chandran , Ankur Dwivedi , "dev@dpdk.org" , Archana Muniganti To: Akhil Goyal , Pablo de Lara Return-path: Received: from mx0b-0016f401.pphosted.com (mx0a-0016f401.pphosted.com [67.231.148.174]) by dpdk.org (Postfix) with ESMTP id 413D94C8D for ; Fri, 1 Mar 2019 19:42:36 +0100 (CET) In-Reply-To: <1551465683-26721-1-git-send-email-anoobj@marvell.com> Content-Language: en-US List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Moving upper level enqueue/dequeue routines to driver. The h/w interface used to submit request has enough differences to substantiate the need for separate routines. Signed-off-by: Ankur Dwivedi Signed-off-by: Anoob Joseph Signed-off-by: Archana Muniganti --- drivers/common/cpt/cpt_common.h | 6 - drivers/common/cpt/cpt_mcode_defines.h | 12 + drivers/common/cpt/cpt_request_mgr.h | 185 --------------- drivers/crypto/octeontx/Makefile | 3 +- drivers/crypto/octeontx/meson.build | 2 +- drivers/crypto/octeontx/otx_cryptodev_hw_access.h | 2 + drivers/crypto/octeontx/otx_cryptodev_ops.c | 262 ++++++++++++++++++= ++-- 7 files changed, 255 insertions(+), 217 deletions(-) delete mode 100644 drivers/common/cpt/cpt_request_mgr.h diff --git a/drivers/common/cpt/cpt_common.h b/drivers/common/cpt/cpt_commo= n.h index 8568d5b..ceb32f2 100644 --- a/drivers/common/cpt/cpt_common.h +++ b/drivers/common/cpt/cpt_common.h @@ -9,12 +9,6 @@ * This file defines common macros and structs */ =20 -/* - * Macros to determine CPT model. Driver makefile will define CPT_MODEL - * accordingly - */ -#define CRYPTO_OCTEONTX 0x1 - #define TIME_IN_RESET_COUNT 5 =20 /* Default command timeout in seconds */ diff --git a/drivers/common/cpt/cpt_mcode_defines.h b/drivers/common/cpt/cp= t_mcode_defines.h index becc14f..c0adbd5 100644 --- a/drivers/common/cpt/cpt_mcode_defines.h +++ b/drivers/common/cpt/cpt_mcode_defines.h @@ -383,4 +383,16 @@ typedef mc_hash_type_t auth_type_t; #define SESS_PRIV(__sess) \ (void *)((uint8_t *)__sess + sizeof(struct cpt_sess_misc)) =20 +/* + * Get the session size + * + * @return + * - session size + */ +static __rte_always_inline unsigned int +cpt_get_session_size(void) +{ + unsigned int ctx_len =3D sizeof(struct cpt_ctx); + return (sizeof(struct cpt_sess_misc) + RTE_ALIGN_CEIL(ctx_len, 8)); +} #endif /* _CPT_MCODE_DEFINES_H_ */ diff --git a/drivers/common/cpt/cpt_request_mgr.h b/drivers/common/cpt/cpt_= request_mgr.h deleted file mode 100644 index 81bddf4..0000000 --- a/drivers/common/cpt/cpt_request_mgr.h +++ /dev/null @@ -1,185 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2018 Cavium, Inc - */ - -#ifndef _CPT_REQUEST_MGR_H_ -#define _CPT_REQUEST_MGR_H_ - -#include -#include - -#include "cpt_common.h" -#include "cpt_mcode_defines.h" - -#if CPT_MODEL =3D=3D CRYPTO_OCTEONTX -#include "../../crypto/octeontx/otx_cryptodev_hw_access.h" -#endif - -/* - * This file defines the agreement between the common layer and the indivi= dual - * crypto drivers for OCTEON TX series. Datapath in otx* directory include= this - * file and all these functions are static inlined for better performance. - * - */ - -/* - * Get the session size - * - * This function is used in the data path. - * - * @return - * - session size - */ -static __rte_always_inline unsigned int -cpt_get_session_size(void) -{ - unsigned int ctx_len =3D sizeof(struct cpt_ctx); - return (sizeof(struct cpt_sess_misc) + RTE_ALIGN_CEIL(ctx_len, 8)); -} - -static __rte_always_inline int32_t __hot -cpt_enqueue_req(struct cpt_instance *instance, struct pending_queue *pqueu= e, - void *req) -{ - struct cpt_request_info *user_req =3D (struct cpt_request_info *)req; - int32_t ret =3D 0; - - if (unlikely(!req)) - return 0; - - if (unlikely(pqueue->pending_count >=3D DEFAULT_CMD_QLEN)) - return -EAGAIN; - - fill_cpt_inst(instance, req); - - CPT_LOG_DP_DEBUG("req: %p op: %p ", req, user_req->op); - - /* Fill time_out cycles */ - user_req->time_out =3D rte_get_timer_cycles() + - DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz(); - user_req->extra_time =3D 0; - - /* Default mode of software queue */ - mark_cpt_inst(instance); - - pqueue->rid_queue[pqueue->enq_tail].rid =3D - (uintptr_t)user_req; - /* We will use soft queue length here to limit - * requests - */ - MOD_INC(pqueue->enq_tail, DEFAULT_CMD_QLEN); - pqueue->pending_count +=3D 1; - - CPT_LOG_DP_DEBUG("Submitted NB cmd with request: %p " - "op: %p", user_req, user_req->op); - - return ret; -} - -static __rte_always_inline int __hot -cpt_pmd_crypto_operation(struct cpt_instance *instance, - struct rte_crypto_op *op, struct pending_queue *pqueue, - uint8_t cpt_driver_id) -{ - struct cpt_sess_misc *sess =3D NULL; - struct rte_crypto_sym_op *sym_op =3D op->sym; - void *prep_req =3D NULL, *mdata =3D NULL; - int ret =3D 0; - uint64_t cpt_op; - struct cpt_vf *cptvf =3D (struct cpt_vf *)instance; - - if (unlikely(op->sess_type =3D=3D RTE_CRYPTO_OP_SESSIONLESS)) { - int sess_len; - - sess_len =3D cpt_get_session_size(); - - sess =3D rte_calloc(__func__, 1, sess_len, 8); - if (!sess) - return -ENOMEM; - - sess->ctx_dma_addr =3D rte_malloc_virt2iova(sess) + - sizeof(struct cpt_sess_misc); - - ret =3D instance_session_cfg(sym_op->xform, (void *)sess); - if (unlikely(ret)) - return -EINVAL; - } else { - sess =3D (struct cpt_sess_misc *) - get_sym_session_private_data(sym_op->session, - cpt_driver_id); - } - - cpt_op =3D sess->cpt_op; - - if (likely(cpt_op & CPT_OP_CIPHER_MASK)) - ret =3D fill_fc_params(op, sess, &cptvf->meta_info, &mdata, - &prep_req); - else - ret =3D fill_digest_params(op, sess, &cptvf->meta_info, - &mdata, &prep_req); - - if (unlikely(ret)) { - CPT_LOG_DP_ERR("prep cryto req : op %p, cpt_op 0x%x " - "ret 0x%x", op, (unsigned int)cpt_op, ret); - return ret; - } - - /* Enqueue prepared instruction to HW */ - ret =3D cpt_enqueue_req(instance, pqueue, prep_req); - - if (unlikely(ret)) { - if (unlikely(ret =3D=3D -EAGAIN)) - goto req_fail; - CPT_LOG_DP_ERR("Error enqueing crypto request : error " - "code %d", ret); - goto req_fail; - } - - return 0; - -req_fail: - if (mdata) - free_op_meta(mdata, cptvf->meta_info.cptvf_meta_pool); - return ret; -} - -static __rte_always_inline int32_t __hot -cpt_dequeue_burst(struct cpt_instance *instance, uint16_t cnt, - void *resp[], uint8_t cc[], struct pending_queue *pqueue) -{ - struct cpt_request_info *user_req; - struct rid *rid_e; - int i, count, pcount; - uint8_t ret; - - pcount =3D pqueue->pending_count; - count =3D (cnt > pcount) ? pcount : cnt; - - for (i =3D 0; i < count; i++) { - rid_e =3D &pqueue->rid_queue[pqueue->deq_head]; - user_req =3D (struct cpt_request_info *)(rid_e->rid); - - if (likely((i+1) < count)) - rte_prefetch_non_temporal((void *)rid_e[1].rid); - - ret =3D check_nb_command_id(user_req, instance); - - if (unlikely(ret =3D=3D ERR_REQ_PENDING)) { - /* Stop checking for completions */ - break; - } - - /* Return completion code and op handle */ - cc[i] =3D (uint8_t)ret; - resp[i] =3D user_req->op; - CPT_LOG_DP_DEBUG("Request %p Op %p completed with code %d", - user_req, user_req->op, ret); - - MOD_INC(pqueue->deq_head, DEFAULT_CMD_QLEN); - pqueue->pending_count -=3D 1; - } - - return i; -} - -#endif /* _CPT_REQUEST_MGR_H_ */ diff --git a/drivers/crypto/octeontx/Makefile b/drivers/crypto/octeontx/Mak= efile index d6412d1..2752cbc 100644 --- a/drivers/crypto/octeontx/Makefile +++ b/drivers/crypto/octeontx/Makefile @@ -20,8 +20,9 @@ LDLIBS +=3D -lrte_common_cpt =20 VPATH +=3D $(RTE_SDK)/drivers/crypto/octeontx =20 -CFLAGS +=3D -O3 -DCPT_MODEL=3DCRYPTO_OCTEONTX +CFLAGS +=3D -O3 CFLAGS +=3D -I$(RTE_SDK)/drivers/common/cpt +CFLAGS +=3D -DALLOW_EXPERIMENTAL_API =20 # PMD code SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) +=3D otx_cryptodev.c diff --git a/drivers/crypto/octeontx/meson.build b/drivers/crypto/octeontx/= meson.build index 6511b40..423737e 100644 --- a/drivers/crypto/octeontx/meson.build +++ b/drivers/crypto/octeontx/meson.build @@ -8,6 +8,7 @@ deps +=3D ['bus_pci'] deps +=3D ['common_cpt'] name =3D 'octeontx_crypto' =20 +allow_experimental_apis =3D true sources =3D files('otx_cryptodev.c', 'otx_cryptodev_capabilities.c', 'otx_cryptodev_hw_access.c', @@ -15,4 +16,3 @@ sources =3D files('otx_cryptodev.c', 'otx_cryptodev_ops.c') =20 includes +=3D include_directories('../../common/cpt') -cflags +=3D '-DCPT_MODEL=3DCRYPTO_OCTEONTX' diff --git a/drivers/crypto/octeontx/otx_cryptodev_hw_access.h b/drivers/cr= ypto/octeontx/otx_cryptodev_hw_access.h index 82b15ee..dea4cba 100644 --- a/drivers/crypto/octeontx/otx_cryptodev_hw_access.h +++ b/drivers/crypto/octeontx/otx_cryptodev_hw_access.h @@ -39,6 +39,8 @@ struct cpt_instance { uint32_t queue_id; uintptr_t rsvd; + struct rte_mempool *sess_mp; + struct rte_mempool *sess_mp_priv; }; =20 struct command_chunk { diff --git a/drivers/crypto/octeontx/otx_cryptodev_ops.c b/drivers/crypto/o= cteontx/otx_cryptodev_ops.c index 6a0cf83..0f9f2a2 100644 --- a/drivers/crypto/octeontx/otx_cryptodev_ops.c +++ b/drivers/crypto/octeontx/otx_cryptodev_ops.c @@ -11,7 +11,6 @@ #include "cpt_pmd_logs.h" #include "cpt_pmd_ops_helper.h" #include "cpt_ucode.h" -#include "cpt_request_mgr.h" =20 #include "otx_cryptodev.h" #include "otx_cryptodev_capabilities.h" @@ -222,6 +221,8 @@ otx_cpt_que_pair_setup(struct rte_cryptodev *dev, } =20 instance->queue_id =3D que_pair_id; + instance->sess_mp =3D qp_conf->mp_session; + instance->sess_mp_priv =3D qp_conf->mp_session_private; dev->data->queue_pairs[que_pair_id] =3D instance; =20 return 0; @@ -340,11 +341,160 @@ otx_cpt_session_clear(struct rte_cryptodev *dev, } } =20 +static __rte_always_inline int32_t __hot +otx_cpt_request_enqueue(struct cpt_instance *instance, + struct pending_queue *pqueue, + void *req) +{ + struct cpt_request_info *user_req =3D (struct cpt_request_info *)req; + + if (unlikely(pqueue->pending_count >=3D DEFAULT_CMD_QLEN)) + return -EAGAIN; + + fill_cpt_inst(instance, req); + + CPT_LOG_DP_DEBUG("req: %p op: %p ", req, user_req->op); + + /* Fill time_out cycles */ + user_req->time_out =3D rte_get_timer_cycles() + + DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz(); + user_req->extra_time =3D 0; + + /* Default mode of software queue */ + mark_cpt_inst(instance); + + pqueue->rid_queue[pqueue->enq_tail].rid =3D (uintptr_t)user_req; + + /* We will use soft queue length here to limit requests */ + MOD_INC(pqueue->enq_tail, DEFAULT_CMD_QLEN); + pqueue->pending_count +=3D 1; + + CPT_LOG_DP_DEBUG("Submitted NB cmd with request: %p " + "op: %p", user_req, user_req->op); + return 0; +} + +static __rte_always_inline int __hot +otx_cpt_enq_single_sym(struct cpt_instance *instance, + struct rte_crypto_op *op, + struct pending_queue *pqueue) +{ + struct cpt_sess_misc *sess; + struct rte_crypto_sym_op *sym_op =3D op->sym; + void *prep_req, *mdata =3D NULL; + int ret =3D 0; + uint64_t cpt_op; + struct cpt_vf *cptvf =3D (struct cpt_vf *)instance; + + sess =3D (struct cpt_sess_misc *) + get_sym_session_private_data(sym_op->session, + otx_cryptodev_driver_id); + + cpt_op =3D sess->cpt_op; + + if (likely(cpt_op & CPT_OP_CIPHER_MASK)) + ret =3D fill_fc_params(op, sess, &cptvf->meta_info, &mdata, + &prep_req); + else + ret =3D fill_digest_params(op, sess, &cptvf->meta_info, + &mdata, &prep_req); + + if (unlikely(ret)) { + CPT_LOG_DP_ERR("prep cryto req : op %p, cpt_op 0x%x " + "ret 0x%x", op, (unsigned int)cpt_op, ret); + return ret; + } + + /* Enqueue prepared instruction to h/w */ + ret =3D otx_cpt_request_enqueue(instance, pqueue, prep_req); + + if (unlikely(ret)) { + /* Buffer allocated for request preparation need to be freed */ + free_op_meta(mdata, cptvf->meta_info.cptvf_meta_pool); + return ret; + } + + return 0; +} + +static __rte_always_inline int __hot +otx_cpt_enq_single_sym_sessless(struct cpt_instance *instance, + struct rte_crypto_op *op, + struct pending_queue *pqueue) +{ + struct cpt_sess_misc *sess; + struct rte_crypto_sym_op *sym_op =3D op->sym; + int ret; + void *sess_t =3D NULL; + void *sess_private_data_t =3D NULL; + + /* Create tmp session */ + + if (rte_mempool_get(instance->sess_mp, (void **)&sess_t)) { + ret =3D -ENOMEM; + goto exit; + } + + if (rte_mempool_get(instance->sess_mp_priv, + (void **)&sess_private_data_t)) { + ret =3D -ENOMEM; + goto free_sess; + } + + sess =3D (struct cpt_sess_misc *)sess_private_data_t; + + sess->ctx_dma_addr =3D rte_mempool_virt2iova(sess) + + sizeof(struct cpt_sess_misc); + + ret =3D instance_session_cfg(sym_op->xform, (void *)sess); + if (unlikely(ret)) { + ret =3D -EINVAL; + goto free_sess_priv; + } + + /* Save tmp session in op */ + + sym_op->session =3D (struct rte_cryptodev_sym_session *)sess_t; + set_sym_session_private_data(sym_op->session, otx_cryptodev_driver_id, + sess_private_data_t); + + /* Enqueue op with the tmp session set */ + ret =3D otx_cpt_enq_single_sym(instance, op, pqueue); + + if (unlikely(ret)) + goto free_sess_priv; + + return 0; + +free_sess_priv: + rte_mempool_put(instance->sess_mp_priv, sess_private_data_t); +free_sess: + rte_mempool_put(instance->sess_mp, sess_t); +exit: + return ret; +} + +static __rte_always_inline int __hot +otx_cpt_enq_single(struct cpt_instance *inst, + struct rte_crypto_op *op, + struct pending_queue *pqueue) +{ + /* Check for the type */ + + if (op->sess_type =3D=3D RTE_CRYPTO_OP_WITH_SESSION) + return otx_cpt_enq_single_sym(inst, op, pqueue); + else if (unlikely(op->sess_type =3D=3D RTE_CRYPTO_OP_SESSIONLESS)) + return otx_cpt_enq_single_sym_sessless(inst, op, pqueue); + + /* Should not reach here */ + return -EINVAL; +} + static uint16_t otx_cpt_pkt_enqueue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_op= s) { struct cpt_instance *instance =3D (struct cpt_instance *)qptr; - uint16_t count =3D 0; + uint16_t count; int ret; struct cpt_vf *cptvf =3D (struct cpt_vf *)instance; struct pending_queue *pqueue =3D &cptvf->pqueue; @@ -355,8 +505,10 @@ otx_cpt_pkt_enqueue(void *qptr, struct rte_crypto_op *= *ops, uint16_t nb_ops) =20 count =3D 0; while (likely(count < nb_ops)) { - ret =3D cpt_pmd_crypto_operation(instance, ops[count], pqueue, - otx_cryptodev_driver_id); + + /* Enqueue single op */ + ret =3D otx_cpt_enq_single(instance, ops[count], pqueue); + if (unlikely(ret)) break; count++; @@ -365,48 +517,110 @@ otx_cpt_pkt_enqueue(void *qptr, struct rte_crypto_op= **ops, uint16_t nb_ops) return count; } =20 +static __rte_always_inline void +otx_cpt_dequeue_post_process(struct rte_crypto_op *cop, uintptr_t *rsp) +{ + /* H/w has returned success */ + cop->status =3D RTE_CRYPTO_OP_STATUS_SUCCESS; + + /* Perform further post processing */ + + if (cop->type =3D=3D RTE_CRYPTO_OP_TYPE_SYMMETRIC) { + /* Check if auth verify need to be completed */ + if (unlikely(rsp[2])) + compl_auth_verify(cop, (uint8_t *)rsp[2], rsp[3]); + return; + } +} + static uint16_t otx_cpt_pkt_dequeue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_op= s) { struct cpt_instance *instance =3D (struct cpt_instance *)qptr; + struct cpt_request_info *user_req; struct cpt_vf *cptvf =3D (struct cpt_vf *)instance; + struct rid *rid_e; + uint8_t cc[nb_ops]; + int i, count, pcount; + uint8_t ret; + int nb_completed; struct pending_queue *pqueue =3D &cptvf->pqueue; - uint16_t nb_completed, i =3D 0; - uint8_t compcode[nb_ops]; + struct rte_crypto_op *cop; + void *metabuf; + uintptr_t *rsp; + + pcount =3D pqueue->pending_count; + count =3D (nb_ops > pcount) ? pcount : nb_ops; + + for (i =3D 0; i < count; i++) { + rid_e =3D &pqueue->rid_queue[pqueue->deq_head]; + user_req =3D (struct cpt_request_info *)(rid_e->rid); + + if (likely((i+1) < count)) + rte_prefetch_non_temporal((void *)rid_e[1].rid); + + ret =3D check_nb_command_id(user_req, instance); + + if (unlikely(ret =3D=3D ERR_REQ_PENDING)) { + /* Stop checking for completions */ + break; + } + + /* Return completion code and op handle */ + cc[i] =3D ret; + ops[i] =3D user_req->op; + + CPT_LOG_DP_DEBUG("Request %p Op %p completed with code %d", + user_req, user_req->op, ret); =20 - nb_completed =3D cpt_dequeue_burst(instance, nb_ops, - (void **)ops, compcode, pqueue); - while (likely(i < nb_completed)) { - struct rte_crypto_op *cop; - void *metabuf; - uintptr_t *rsp; - uint8_t status; + MOD_INC(pqueue->deq_head, DEFAULT_CMD_QLEN); + pqueue->pending_count -=3D 1; + } + + nb_completed =3D i; + + for (i =3D 0; i < nb_completed; i++) { =20 rsp =3D (void *)ops[i]; - status =3D compcode[i]; + if (likely((i + 1) < nb_completed)) rte_prefetch0(ops[i+1]); + metabuf =3D (void *)rsp[0]; cop =3D (void *)rsp[1]; =20 ops[i] =3D cop; =20 - if (likely(status =3D=3D 0)) { - if (likely(!rsp[2])) - cop->status =3D - RTE_CRYPTO_OP_STATUS_SUCCESS; - else - compl_auth_verify(cop, (uint8_t *)rsp[2], - rsp[3]); - } else if (status =3D=3D ERR_GC_ICV_MISCOMPARE) { - /*auth data mismatch */ + /* Check completion code */ + + if (likely(cc[i] =3D=3D 0)) { + /* H/w success pkt. Post process */ + otx_cpt_dequeue_post_process(cop, rsp); + } else if (cc[i] =3D=3D ERR_GC_ICV_MISCOMPARE) { + /* auth data mismatch */ cop->status =3D RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } else { + /* Error */ cop->status =3D RTE_CRYPTO_OP_STATUS_ERROR; } + + if (unlikely(cop->sess_type =3D=3D RTE_CRYPTO_OP_SESSIONLESS)) { + void *sess_private_data_t =3D + get_sym_session_private_data(cop->sym->session, + otx_cryptodev_driver_id); + memset(sess_private_data_t, 0, + cpt_get_session_size()); + memset(cop->sym->session, 0, + rte_cryptodev_sym_get_existing_header_session_size( + cop->sym->session)); + rte_mempool_put(instance->sess_mp_priv, + sess_private_data_t); + rte_mempool_put(instance->sess_mp, cop->sym->session); + cop->sym->session =3D NULL; + } free_op_meta(metabuf, cptvf->meta_info.cptvf_meta_pool); - i++; } + return nb_completed; } =20 --=20 2.7.4