* [PATCH] crypto: hisilicon/zip - add backlog support for zip
2026-05-15 11:45 [PATCH 0/3] crypto: hisilicon - improve backlog handling ZongYu Wu
@ 2026-05-15 11:45 ` ZongYu Wu
2026-05-22 10:32 ` huangchenghai
2026-05-15 11:46 ` [PATCH 2/3] crypto: hisilicon/sec2 - fix UAF in sec_alg_send_backlog ZongYu Wu
2026-05-15 11:46 ` [PATCH 3/3] crypto: hisilicon/hpre - implement full backlog support for hpre driver ZongYu Wu
2 siblings, 1 reply; 5+ messages in thread
From: ZongYu Wu @ 2026-05-15 11:45 UTC (permalink / raw)
To: herbert, davem
Cc: linux-kernel, linux-crypto, fanghao11, liulongfang, qianweili,
wangzhou1
From: Chenghai Huang <huangchenghai2@huawei.com>
When the hardware queue is busy, requests are now queued instead of
being failed immediately. Queued requests are retried when earlier
requests complete, which prevents transient failures under heavy load.
The backlog path also provides a fallback mechanism while the hardware
is temporarily unavailable, such as during device reset.
Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com>
Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com>
---
drivers/crypto/hisilicon/zip/zip_crypto.c | 284 ++++++++++++++--------
1 file changed, 180 insertions(+), 104 deletions(-)
diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c
index 70adde049b53..ff287251218e 100644
--- a/drivers/crypto/hisilicon/zip/zip_crypto.c
+++ b/drivers/crypto/hisilicon/zip/zip_crypto.c
@@ -28,6 +28,7 @@
#define HZIP_ALG_DEFLATE GENMASK(5, 4)
#define HZIP_ALG_LZ4 BIT(8)
+#define HZIP_INVAL_REQ_ID ((u16)0xFFFF)
static DEFINE_MUTEX(zip_algs_lock);
static unsigned int zip_available_devs;
@@ -55,11 +56,11 @@ struct hisi_zip_req {
dma_addr_t dma_src;
dma_addr_t dma_dst;
struct hisi_zip_qp_ctx *qp_ctx;
+ struct list_head list;
u16 req_id;
};
struct hisi_zip_req_q {
- struct hisi_zip_req *q;
unsigned long *req_bitmap;
spinlock_t req_lock;
u16 size;
@@ -135,42 +136,42 @@ static int hisi_zip_fallback_do_work(struct acomp_req *acomp_req, bool is_decomp
return ret;
}
-static struct hisi_zip_req *hisi_zip_create_req(struct hisi_zip_qp_ctx *qp_ctx,
- struct acomp_req *req)
+static int hisi_zip_create_req(struct hisi_zip_req *req)
{
+ struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx;
struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
- struct hisi_zip_req *q = req_q->q;
- struct hisi_zip_req *req_cache;
int req_id;
+ /* Check whether any request is being queued */
+ if (!list_empty(&qp_ctx->qp->backlog.list))
+ return -EBUSY;
+
spin_lock(&req_q->req_lock);
req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size);
if (req_id >= req_q->size) {
spin_unlock(&req_q->req_lock);
dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n");
- return ERR_PTR(-EAGAIN);
+ return -EBUSY;
}
set_bit(req_id, req_q->req_bitmap);
spin_unlock(&req_q->req_lock);
- req_cache = q + req_id;
- req_cache->req_id = req_id;
- req_cache->req = req;
- req_cache->qp_ctx = qp_ctx;
+ req->req_id = req_id;
- return req_cache;
+ return 0;
}
-static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx,
- struct hisi_zip_req *req)
+static void hisi_zip_remove_req(struct hisi_zip_req *req)
{
- struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
+ struct hisi_zip_req_q *req_q = &req->qp_ctx->req_q;
spin_lock(&req_q->req_lock);
clear_bit(req->req_id, req_q->req_bitmap);
spin_unlock(&req_q->req_lock);
+
+ req->req_id = HZIP_INVAL_REQ_ID;
}
static void hisi_zip_fill_addr(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req)
@@ -247,19 +248,21 @@ static void hisi_zip_fill_sqe(struct hisi_zip_ctx *ctx, struct hisi_zip_sqe *sqe
ops->fill_sqe_type(sqe, ops->sqe_type);
}
-static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx,
- struct hisi_zip_req *req)
+static void hisi_zip_enqueue_backlog(struct hisi_zip_req *req)
+{
+ struct hisi_qp *qp = req->qp_ctx->qp;
+
+ spin_lock_bh(&qp->backlog.lock);
+ list_add_tail(&req->list, &qp->backlog.list);
+ spin_unlock_bh(&qp->backlog.lock);
+}
+
+static int hisi_zip_map_req_buffers(struct hisi_zip_req *req)
{
+ struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx;
struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool;
- struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
+ struct device *dev = &qp_ctx->qp->qm->pdev->dev;
struct acomp_req *a_req = req->req;
- struct hisi_qp *qp = qp_ctx->qp;
- struct device *dev = &qp->qm->pdev->dev;
- struct hisi_zip_sqe zip_sqe;
- int ret;
-
- if (unlikely(!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen))
- return -EINVAL;
req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool,
req->req_id << 1, &req->dma_src,
@@ -274,33 +277,110 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx,
(req->req_id << 1) + 1,
&req->dma_dst, DMA_FROM_DEVICE);
if (IS_ERR(req->hw_dst)) {
- ret = PTR_ERR(req->hw_dst);
- dev_err(dev, "failed to map the dst buffer to hw sgl (%d)!\n",
- ret);
- goto err_unmap_input;
+ dev_err(dev, "failed to map the dst buffer to hw sgl (%ld)!\n",
+ PTR_ERR(req->hw_dst));
+ hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src, DMA_TO_DEVICE);
+ return PTR_ERR(req->hw_dst);
}
+ return 0;
+}
+
+static void hisi_zip_unmap_req_buffers(struct hisi_zip_req *req)
+{
+ struct device *dev = &req->qp_ctx->qp->qm->pdev->dev;
+ struct acomp_req *a_req = req->req;
+
+ hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst, DMA_FROM_DEVICE);
+ hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src, DMA_TO_DEVICE);
+}
+
+static int hisi_zip_do_work(struct hisi_zip_req *req)
+{
+ struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx;
+ struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
+ struct hisi_qp *qp = qp_ctx->qp;
+ struct hisi_zip_sqe zip_sqe;
+ int ret;
+
hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp_ctx->req_type, req);
/* send command to start a task */
- atomic64_inc(&dfx->send_cnt);
ret = hisi_qp_send(qp, &zip_sqe);
- if (unlikely(ret < 0)) {
- atomic64_inc(&dfx->send_busy_cnt);
- ret = -EAGAIN;
- dev_dbg_ratelimited(dev, "failed to send request!\n");
- goto err_unmap_output;
+ if (likely(!ret)) {
+ atomic64_inc(&dfx->send_cnt);
+ return -EINPROGRESS;
}
- return -EINPROGRESS;
+ if (ret == -EBUSY)
+ atomic64_inc(&dfx->send_busy_cnt);
-err_unmap_output:
- hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst, DMA_FROM_DEVICE);
-err_unmap_input:
- hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src, DMA_TO_DEVICE);
return ret;
}
+static void hisi_zip_send_backlog_soft(struct hisi_zip_qp_ctx *qp_ctx)
+{
+ bool is_decomp = qp_ctx->qp->alg_type;
+ struct hisi_zip_req *req, *tmp;
+ int ret;
+
+ list_for_each_entry_safe(req, tmp, &qp_ctx->qp->backlog.list, list) {
+ list_del(&req->list);
+
+ if (req->req_id != HZIP_INVAL_REQ_ID) {
+ hisi_zip_unmap_req_buffers(req);
+ hisi_zip_remove_req(req);
+ }
+
+ ret = hisi_zip_fallback_do_work(req->req, is_decomp);
+
+ /* Wake up the busy thread first, then return the errno. */
+ if (req->req->base.complete) {
+ acomp_request_complete(req->req, -EINPROGRESS);
+ acomp_request_complete(req->req, ret);
+ }
+ }
+}
+
+static void hisi_zip_send_backlog(struct hisi_qp *qp)
+{
+ struct hisi_zip_req *req, *tmp;
+ int ret;
+
+ spin_lock_bh(&qp->backlog.lock);
+ list_for_each_entry_safe(req, tmp, &qp->backlog.list, list) {
+ if (req->req_id == HZIP_INVAL_REQ_ID) {
+ ret = hisi_zip_create_req(req);
+ if (ret)
+ continue;
+
+ ret = hisi_zip_map_req_buffers(req);
+ if (unlikely(ret)) {
+ hisi_zip_remove_req(req);
+ hisi_zip_send_backlog_soft(req->qp_ctx);
+ goto unlock;
+ }
+ }
+
+ ret = hisi_zip_do_work(req);
+ switch (ret) {
+ case -EINPROGRESS:
+ list_del(&req->list);
+ if (req->req->base.complete)
+ acomp_request_complete(req->req, -EINPROGRESS);
+ break;
+ case -EBUSY:
+ goto unlock;
+ default:
+ hisi_zip_send_backlog_soft(req->qp_ctx);
+ goto unlock;
+ }
+ }
+
+unlock:
+ spin_unlock_bh(&qp->backlog.lock);
+}
+
static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe)
{
return sqe->dw3 & HZIP_BD_STATUS_M;
@@ -333,73 +413,89 @@ static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data)
err = -EIO;
}
- hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst, DMA_FROM_DEVICE);
- hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src, DMA_TO_DEVICE);
+ hisi_zip_unmap_req_buffers(req);
acomp_req->dlen = ops->get_dstlen(sqe);
+ hisi_zip_remove_req(req);
if (acomp_req->base.complete)
acomp_request_complete(acomp_req, err);
- hisi_zip_remove_req(qp_ctx, req);
+ hisi_zip_send_backlog(qp);
}
-static int hisi_zip_acompress(struct acomp_req *acomp_req)
+static int hisi_zip_do_comp(struct hisi_zip_req *req)
{
+ struct acomp_req *acomp_req = req->req;
struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
- struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP];
- struct hisi_zip_req *req;
- struct device *dev;
+ struct hisi_qp *qp = req->qp_ctx->qp;
int ret;
- if (ctx->fallback)
- return hisi_zip_fallback_do_work(acomp_req, 0);
-
- dev = &qp_ctx->qp->qm->pdev->dev;
+ if (unlikely(!acomp_req->src || !acomp_req->slen ||
+ !acomp_req->dst || !acomp_req->dlen))
+ return -EINVAL;
- req = hisi_zip_create_req(qp_ctx, acomp_req);
- if (IS_ERR(req))
- return PTR_ERR(req);
+ if (ctx->fallback)
+ return hisi_zip_fallback_do_work(acomp_req, qp->alg_type);
+
+ ret = hisi_zip_create_req(req);
+ if (ret && (acomp_req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
+ /* all req bitmaps are used add to backlog list */
+ req->req_id = HZIP_INVAL_REQ_ID;
+ hisi_zip_enqueue_backlog(req);
+ return -EBUSY;
+ } else if (unlikely(ret)) {
+ return -ENOSPC;
+ }
- ret = hisi_zip_do_work(qp_ctx, req);
- if (unlikely(ret != -EINPROGRESS)) {
- dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret);
- hisi_zip_remove_req(qp_ctx, req);
+ ret = hisi_zip_map_req_buffers(req);
+ if (unlikely(ret))
+ goto remove_req;
+
+ ret = hisi_zip_do_work(req);
+ if (ret == -EBUSY && (acomp_req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
+ /* hardwre busy add to backlog list */
+ hisi_zip_enqueue_backlog(req);
+ } else if (unlikely(ret != -EINPROGRESS)) {
+ dev_info_ratelimited(&qp->qm->pdev->dev,
+ "failed to do %scompress (%d)!\n",
+ qp->alg_type ? "de" : "", ret);
+ ret = -ENOSPC;
+ goto unmap_req;
}
return ret;
+
+unmap_req:
+ hisi_zip_unmap_req_buffers(req);
+remove_req:
+ hisi_zip_remove_req(req);
+ return ret;
}
-static int hisi_zip_adecompress(struct acomp_req *acomp_req)
+static int hisi_zip_acompress(struct acomp_req *acomp_req)
{
struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
- struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP];
- struct hisi_zip_req *req;
- struct device *dev;
- int ret;
-
- if (ctx->fallback)
- return hisi_zip_fallback_do_work(acomp_req, 1);
+ struct hisi_zip_req *req = acomp_request_ctx(acomp_req);
- dev = &qp_ctx->qp->qm->pdev->dev;
-
- req = hisi_zip_create_req(qp_ctx, acomp_req);
- if (IS_ERR(req))
- return PTR_ERR(req);
+ req->req = acomp_req;
+ req->qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP];
+ return hisi_zip_do_comp(req);
+}
- ret = hisi_zip_do_work(qp_ctx, req);
- if (unlikely(ret != -EINPROGRESS)) {
- dev_info_ratelimited(dev, "failed to do decompress (%d)!\n",
- ret);
- hisi_zip_remove_req(qp_ctx, req);
- }
+static int hisi_zip_adecompress(struct acomp_req *acomp_req)
+{
+ struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
+ struct hisi_zip_req *req = acomp_request_ctx(acomp_req);
- return ret;
+ req->req = acomp_req;
+ req->qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP];
+ return hisi_zip_do_comp(req);
}
static int hisi_zip_decompress(struct acomp_req *acomp_req)
{
- return hisi_zip_fallback_do_work(acomp_req, 1);
+ return hisi_zip_fallback_do_work(acomp_req, HZIP_ALG_TYPE_DECOMP);
}
static const struct hisi_zip_sqe_ops hisi_zip_ops = {
@@ -463,7 +559,7 @@ static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx)
{
u16 q_depth = ctx->qp_ctx[0].qp->sq_depth;
struct hisi_zip_req_q *req_q;
- int i, ret;
+ int i;
for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
req_q = &ctx->qp_ctx[i].req_q;
@@ -471,43 +567,21 @@ static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx)
req_q->req_bitmap = bitmap_zalloc(req_q->size, GFP_KERNEL);
if (!req_q->req_bitmap) {
- ret = -ENOMEM;
- if (i == 0)
- return ret;
-
- goto err_free_comp_q;
+ bitmap_free(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap);
+ return -ENOMEM;
}
spin_lock_init(&req_q->req_lock);
-
- req_q->q = kzalloc_objs(struct hisi_zip_req, req_q->size);
- if (!req_q->q) {
- ret = -ENOMEM;
- if (i == 0)
- goto err_free_comp_bitmap;
- else
- goto err_free_decomp_bitmap;
- }
}
return 0;
-
-err_free_decomp_bitmap:
- bitmap_free(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap);
-err_free_comp_q:
- kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q);
-err_free_comp_bitmap:
- bitmap_free(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap);
- return ret;
}
static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx)
{
int i;
- for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
- kfree(ctx->qp_ctx[i].req_q.q);
+ for (i = 0; i < HZIP_CTX_Q_NUM; i++)
bitmap_free(ctx->qp_ctx[i].req_q.req_bitmap);
- }
}
static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx)
@@ -620,6 +694,7 @@ static struct acomp_alg hisi_zip_acomp_deflate = {
.cra_module = THIS_MODULE,
.cra_priority = HZIP_ALG_PRIORITY,
.cra_ctxsize = sizeof(struct hisi_zip_ctx),
+ .cra_reqsize = sizeof(struct hisi_zip_req),
}
};
@@ -658,6 +733,7 @@ static struct acomp_alg hisi_zip_acomp_lz4 = {
.cra_module = THIS_MODULE,
.cra_priority = HZIP_ALG_PRIORITY,
.cra_ctxsize = sizeof(struct hisi_zip_ctx),
+ .cra_reqsize = sizeof(struct hisi_zip_req),
}
};
--
2.43.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* Re: [PATCH] crypto: hisilicon/zip - add backlog support for zip
2026-05-15 11:45 ` [PATCH] crypto: hisilicon/zip - add backlog support for zip ZongYu Wu
@ 2026-05-22 10:32 ` huangchenghai
0 siblings, 0 replies; 5+ messages in thread
From: huangchenghai @ 2026-05-22 10:32 UTC (permalink / raw)
To: ZongYu Wu, herbert, davem
Cc: linux-kernel, linux-crypto, fanghao11, liulongfang, qianweili,
wangzhou1
在 2026/5/15 19:45, ZongYu Wu 写道:
> From: Chenghai Huang <huangchenghai2@huawei.com>
>
> When the hardware queue is busy, requests are now queued instead of
> being failed immediately. Queued requests are retried when earlier
> requests complete, which prevents transient failures under heavy load.
>
> The backlog path also provides a fallback mechanism while the hardware
> is temporarily unavailable, such as during device reset.
>
> Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com>
> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com>
> ---
> drivers/crypto/hisilicon/zip/zip_crypto.c | 284 ++++++++++++++--------
> 1 file changed, 180 insertions(+), 104 deletions(-)
>
> diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c
> index 70adde049b53..ff287251218e 100644
> --- a/drivers/crypto/hisilicon/zip/zip_crypto.c
> +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c
> @@ -28,6 +28,7 @@
>
> #define HZIP_ALG_DEFLATE GENMASK(5, 4)
> #define HZIP_ALG_LZ4 BIT(8)
> +#define HZIP_INVAL_REQ_ID ((u16)0xFFFF)
>
> static DEFINE_MUTEX(zip_algs_lock);
> static unsigned int zip_available_devs;
> @@ -55,11 +56,11 @@ struct hisi_zip_req {
> dma_addr_t dma_src;
> dma_addr_t dma_dst;
> struct hisi_zip_qp_ctx *qp_ctx;
> + struct list_head list;
> u16 req_id;
> };
>
> struct hisi_zip_req_q {
> - struct hisi_zip_req *q;
> unsigned long *req_bitmap;
> spinlock_t req_lock;
> u16 size;
> @@ -135,42 +136,42 @@ static int hisi_zip_fallback_do_work(struct acomp_req *acomp_req, bool is_decomp
> return ret;
> }
>
> -static struct hisi_zip_req *hisi_zip_create_req(struct hisi_zip_qp_ctx *qp_ctx,
> - struct acomp_req *req)
> +static int hisi_zip_create_req(struct hisi_zip_req *req)
> {
> + struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx;
> struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
> - struct hisi_zip_req *q = req_q->q;
> - struct hisi_zip_req *req_cache;
> int req_id;
>
> + /* Check whether any request is being queued */
> + if (!list_empty(&qp_ctx->qp->backlog.list))
> + return -EBUSY;
> +
> spin_lock(&req_q->req_lock);
>
> req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size);
> if (req_id >= req_q->size) {
> spin_unlock(&req_q->req_lock);
> dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n");
> - return ERR_PTR(-EAGAIN);
> + return -EBUSY;
> }
> set_bit(req_id, req_q->req_bitmap);
>
> spin_unlock(&req_q->req_lock);
>
> - req_cache = q + req_id;
> - req_cache->req_id = req_id;
> - req_cache->req = req;
> - req_cache->qp_ctx = qp_ctx;
> + req->req_id = req_id;
>
> - return req_cache;
> + return 0;
> }
>
> -static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx,
> - struct hisi_zip_req *req)
> +static void hisi_zip_remove_req(struct hisi_zip_req *req)
> {
> - struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
> + struct hisi_zip_req_q *req_q = &req->qp_ctx->req_q;
>
> spin_lock(&req_q->req_lock);
> clear_bit(req->req_id, req_q->req_bitmap);
> spin_unlock(&req_q->req_lock);
> +
> + req->req_id = HZIP_INVAL_REQ_ID;
> }
>
> static void hisi_zip_fill_addr(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req)
> @@ -247,19 +248,21 @@ static void hisi_zip_fill_sqe(struct hisi_zip_ctx *ctx, struct hisi_zip_sqe *sqe
> ops->fill_sqe_type(sqe, ops->sqe_type);
> }
>
> -static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx,
> - struct hisi_zip_req *req)
> +static void hisi_zip_enqueue_backlog(struct hisi_zip_req *req)
> +{
> + struct hisi_qp *qp = req->qp_ctx->qp;
> +
> + spin_lock_bh(&qp->backlog.lock);
> + list_add_tail(&req->list, &qp->backlog.list);
> + spin_unlock_bh(&qp->backlog.lock);
> +}
> +
> +static int hisi_zip_map_req_buffers(struct hisi_zip_req *req)
> {
> + struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx;
> struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool;
> - struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
> + struct device *dev = &qp_ctx->qp->qm->pdev->dev;
> struct acomp_req *a_req = req->req;
> - struct hisi_qp *qp = qp_ctx->qp;
> - struct device *dev = &qp->qm->pdev->dev;
> - struct hisi_zip_sqe zip_sqe;
> - int ret;
> -
> - if (unlikely(!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen))
> - return -EINVAL;
>
> req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool,
> req->req_id << 1, &req->dma_src,
> @@ -274,33 +277,110 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx,
> (req->req_id << 1) + 1,
> &req->dma_dst, DMA_FROM_DEVICE);
> if (IS_ERR(req->hw_dst)) {
> - ret = PTR_ERR(req->hw_dst);
> - dev_err(dev, "failed to map the dst buffer to hw sgl (%d)!\n",
> - ret);
> - goto err_unmap_input;
> + dev_err(dev, "failed to map the dst buffer to hw sgl (%ld)!\n",
> + PTR_ERR(req->hw_dst));
> + hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src, DMA_TO_DEVICE);
> + return PTR_ERR(req->hw_dst);
> }
>
> + return 0;
> +}
> +
> +static void hisi_zip_unmap_req_buffers(struct hisi_zip_req *req)
> +{
> + struct device *dev = &req->qp_ctx->qp->qm->pdev->dev;
> + struct acomp_req *a_req = req->req;
> +
> + hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst, DMA_FROM_DEVICE);
> + hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src, DMA_TO_DEVICE);
> +}
> +
> +static int hisi_zip_do_work(struct hisi_zip_req *req)
> +{
> + struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx;
> + struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
> + struct hisi_qp *qp = qp_ctx->qp;
> + struct hisi_zip_sqe zip_sqe;
> + int ret;
> +
> hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp_ctx->req_type, req);
>
> /* send command to start a task */
> - atomic64_inc(&dfx->send_cnt);
> ret = hisi_qp_send(qp, &zip_sqe);
> - if (unlikely(ret < 0)) {
> - atomic64_inc(&dfx->send_busy_cnt);
> - ret = -EAGAIN;
> - dev_dbg_ratelimited(dev, "failed to send request!\n");
> - goto err_unmap_output;
> + if (likely(!ret)) {
> + atomic64_inc(&dfx->send_cnt);
> + return -EINPROGRESS;
> }
>
> - return -EINPROGRESS;
> + if (ret == -EBUSY)
> + atomic64_inc(&dfx->send_busy_cnt);
>
> -err_unmap_output:
> - hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst, DMA_FROM_DEVICE);
> -err_unmap_input:
> - hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src, DMA_TO_DEVICE);
> return ret;
> }
>
> +static void hisi_zip_send_backlog_soft(struct hisi_zip_qp_ctx *qp_ctx)
> +{
> + bool is_decomp = qp_ctx->qp->alg_type;
> + struct hisi_zip_req *req, *tmp;
> + int ret;
> +
> + list_for_each_entry_safe(req, tmp, &qp_ctx->qp->backlog.list, list) {
> + list_del(&req->list);
> +
> + if (req->req_id != HZIP_INVAL_REQ_ID) {
> + hisi_zip_unmap_req_buffers(req);
> + hisi_zip_remove_req(req);
> + }
> +
> + ret = hisi_zip_fallback_do_work(req->req, is_decomp);
> +
> + /* Wake up the busy thread first, then return the errno. */
> + if (req->req->base.complete) {
> + acomp_request_complete(req->req, -EINPROGRESS);
> + acomp_request_complete(req->req, ret);
> + }
> + }
> +}
> +
> +static void hisi_zip_send_backlog(struct hisi_qp *qp)
> +{
> + struct hisi_zip_req *req, *tmp;
> + int ret;
> +
> + spin_lock_bh(&qp->backlog.lock);
> + list_for_each_entry_safe(req, tmp, &qp->backlog.list, list) {
> + if (req->req_id == HZIP_INVAL_REQ_ID) {
> + ret = hisi_zip_create_req(req);
> + if (ret)
> + continue;
> +
> + ret = hisi_zip_map_req_buffers(req);
> + if (unlikely(ret)) {
> + hisi_zip_remove_req(req);
> + hisi_zip_send_backlog_soft(req->qp_ctx);
> + goto unlock;
> + }
> + }
> +
> + ret = hisi_zip_do_work(req);
> + switch (ret) {
> + case -EINPROGRESS:
> + list_del(&req->list);
> + if (req->req->base.complete)
> + acomp_request_complete(req->req, -EINPROGRESS);
> + break;
> + case -EBUSY:
> + goto unlock;
> + default:
> + hisi_zip_send_backlog_soft(req->qp_ctx);
> + goto unlock;
> + }
> + }
> +
> +unlock:
> + spin_unlock_bh(&qp->backlog.lock);
> +}
> +
> static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe)
> {
> return sqe->dw3 & HZIP_BD_STATUS_M;
> @@ -333,73 +413,89 @@ static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data)
> err = -EIO;
> }
>
> - hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst, DMA_FROM_DEVICE);
> - hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src, DMA_TO_DEVICE);
> + hisi_zip_unmap_req_buffers(req);
>
> acomp_req->dlen = ops->get_dstlen(sqe);
> + hisi_zip_remove_req(req);
>
> if (acomp_req->base.complete)
> acomp_request_complete(acomp_req, err);
>
> - hisi_zip_remove_req(qp_ctx, req);
> + hisi_zip_send_backlog(qp);
> }
>
> -static int hisi_zip_acompress(struct acomp_req *acomp_req)
> +static int hisi_zip_do_comp(struct hisi_zip_req *req)
> {
> + struct acomp_req *acomp_req = req->req;
> struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
> - struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP];
> - struct hisi_zip_req *req;
> - struct device *dev;
> + struct hisi_qp *qp = req->qp_ctx->qp;
> int ret;
>
> - if (ctx->fallback)
> - return hisi_zip_fallback_do_work(acomp_req, 0);
> -
> - dev = &qp_ctx->qp->qm->pdev->dev;
> + if (unlikely(!acomp_req->src || !acomp_req->slen ||
> + !acomp_req->dst || !acomp_req->dlen))
> + return -EINVAL;
>
> - req = hisi_zip_create_req(qp_ctx, acomp_req);
> - if (IS_ERR(req))
> - return PTR_ERR(req);
> + if (ctx->fallback)
> + return hisi_zip_fallback_do_work(acomp_req, qp->alg_type);
Sorry. I found a problem after review again: when the QP allocation
fails, we cannot obtain the service
type through qp->alg_type, because qp is not valid and ctx->fallback
will true at that point.
I'll fix this and send a v2. Sorry again for the mistake. Please ignore
this patch.
Best regards,
Chenghai
> +
> + ret = hisi_zip_create_req(req);
> + if (ret && (acomp_req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
> + /* all req bitmaps are used add to backlog list */
> + req->req_id = HZIP_INVAL_REQ_ID;
> + hisi_zip_enqueue_backlog(req);
> + return -EBUSY;
> + } else if (unlikely(ret)) {
> + return -ENOSPC;
> + }
>
> - ret = hisi_zip_do_work(qp_ctx, req);
> - if (unlikely(ret != -EINPROGRESS)) {
> - dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret);
> - hisi_zip_remove_req(qp_ctx, req);
> + ret = hisi_zip_map_req_buffers(req);
> + if (unlikely(ret))
> + goto remove_req;
> +
> + ret = hisi_zip_do_work(req);
> + if (ret == -EBUSY && (acomp_req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
> + /* hardwre busy add to backlog list */
> + hisi_zip_enqueue_backlog(req);
> + } else if (unlikely(ret != -EINPROGRESS)) {
> + dev_info_ratelimited(&qp->qm->pdev->dev,
> + "failed to do %scompress (%d)!\n",
> + qp->alg_type ? "de" : "", ret);
> + ret = -ENOSPC;
> + goto unmap_req;
> }
>
> return ret;
> +
> +unmap_req:
> + hisi_zip_unmap_req_buffers(req);
> +remove_req:
> + hisi_zip_remove_req(req);
> + return ret;
> }
>
> -static int hisi_zip_adecompress(struct acomp_req *acomp_req)
> +static int hisi_zip_acompress(struct acomp_req *acomp_req)
> {
> struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
> - struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP];
> - struct hisi_zip_req *req;
> - struct device *dev;
> - int ret;
> -
> - if (ctx->fallback)
> - return hisi_zip_fallback_do_work(acomp_req, 1);
> + struct hisi_zip_req *req = acomp_request_ctx(acomp_req);
>
> - dev = &qp_ctx->qp->qm->pdev->dev;
> -
> - req = hisi_zip_create_req(qp_ctx, acomp_req);
> - if (IS_ERR(req))
> - return PTR_ERR(req);
> + req->req = acomp_req;
> + req->qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP];
> + return hisi_zip_do_comp(req);
> +}
>
> - ret = hisi_zip_do_work(qp_ctx, req);
> - if (unlikely(ret != -EINPROGRESS)) {
> - dev_info_ratelimited(dev, "failed to do decompress (%d)!\n",
> - ret);
> - hisi_zip_remove_req(qp_ctx, req);
> - }
> +static int hisi_zip_adecompress(struct acomp_req *acomp_req)
> +{
> + struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
> + struct hisi_zip_req *req = acomp_request_ctx(acomp_req);
>
> - return ret;
> + req->req = acomp_req;
> + req->qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP];
> + return hisi_zip_do_comp(req);
> }
>
> static int hisi_zip_decompress(struct acomp_req *acomp_req)
> {
> - return hisi_zip_fallback_do_work(acomp_req, 1);
> + return hisi_zip_fallback_do_work(acomp_req, HZIP_ALG_TYPE_DECOMP);
> }
>
> static const struct hisi_zip_sqe_ops hisi_zip_ops = {
> @@ -463,7 +559,7 @@ static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx)
> {
> u16 q_depth = ctx->qp_ctx[0].qp->sq_depth;
> struct hisi_zip_req_q *req_q;
> - int i, ret;
> + int i;
>
> for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
> req_q = &ctx->qp_ctx[i].req_q;
> @@ -471,43 +567,21 @@ static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx)
>
> req_q->req_bitmap = bitmap_zalloc(req_q->size, GFP_KERNEL);
> if (!req_q->req_bitmap) {
> - ret = -ENOMEM;
> - if (i == 0)
> - return ret;
> -
> - goto err_free_comp_q;
> + bitmap_free(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap);
> + return -ENOMEM;
> }
> spin_lock_init(&req_q->req_lock);
> -
> - req_q->q = kzalloc_objs(struct hisi_zip_req, req_q->size);
> - if (!req_q->q) {
> - ret = -ENOMEM;
> - if (i == 0)
> - goto err_free_comp_bitmap;
> - else
> - goto err_free_decomp_bitmap;
> - }
> }
>
> return 0;
> -
> -err_free_decomp_bitmap:
> - bitmap_free(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap);
> -err_free_comp_q:
> - kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q);
> -err_free_comp_bitmap:
> - bitmap_free(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap);
> - return ret;
> }
>
> static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx)
> {
> int i;
>
> - for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
> - kfree(ctx->qp_ctx[i].req_q.q);
> + for (i = 0; i < HZIP_CTX_Q_NUM; i++)
> bitmap_free(ctx->qp_ctx[i].req_q.req_bitmap);
> - }
> }
>
> static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx)
> @@ -620,6 +694,7 @@ static struct acomp_alg hisi_zip_acomp_deflate = {
> .cra_module = THIS_MODULE,
> .cra_priority = HZIP_ALG_PRIORITY,
> .cra_ctxsize = sizeof(struct hisi_zip_ctx),
> + .cra_reqsize = sizeof(struct hisi_zip_req),
> }
> };
>
> @@ -658,6 +733,7 @@ static struct acomp_alg hisi_zip_acomp_lz4 = {
> .cra_module = THIS_MODULE,
> .cra_priority = HZIP_ALG_PRIORITY,
> .cra_ctxsize = sizeof(struct hisi_zip_ctx),
> + .cra_reqsize = sizeof(struct hisi_zip_req),
> }
> };
>
^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH 2/3] crypto: hisilicon/sec2 - fix UAF in sec_alg_send_backlog
2026-05-15 11:45 [PATCH 0/3] crypto: hisilicon - improve backlog handling ZongYu Wu
2026-05-15 11:45 ` [PATCH] crypto: hisilicon/zip - add backlog support for zip ZongYu Wu
@ 2026-05-15 11:46 ` ZongYu Wu
2026-05-15 11:46 ` [PATCH 3/3] crypto: hisilicon/hpre - implement full backlog support for hpre driver ZongYu Wu
2 siblings, 0 replies; 5+ messages in thread
From: ZongYu Wu @ 2026-05-15 11:46 UTC (permalink / raw)
To: herbert, davem
Cc: linux-kernel, linux-crypto, fanghao11, liulongfang, qianweili,
wangzhou1
From: Wenkai Lin <linwenkai6@hisilicon.com>
After crypto_request_complete() is invoked, the crypto core may
immediately free the request structure and its associated tfm context.
Consequently, the sec_ctx and qp_ctx are also released.
However, sec_alg_send_backlog() can still attempt to access these
structures when processing queued requests, resulting in a
use-after-free (UAF) bug.
Fix this by accessing the backlog list through the long-term qp memory
and using the ctx memory only when the backlog list is not empty.
Fixes: f0ae287c5045 ("crypto: hisilicon/sec2 - implement full backlog mode for sec")
Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com>
Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com>
---
drivers/crypto/hisilicon/sec2/sec_crypto.c | 23 +++++++++++-----------
1 file changed, 11 insertions(+), 12 deletions(-)
diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c
index 2471a4dd0b50..7f2c81170431 100644
--- a/drivers/crypto/hisilicon/sec2/sec_crypto.c
+++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c
@@ -234,13 +234,15 @@ static int qp_send_message(struct sec_req *req)
return -EINPROGRESS;
}
-static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
+static void sec_alg_send_backlog_soft(struct hisi_qp *qp)
{
struct sec_req *req, *tmp;
+ struct sec_ctx *ctx;
int ret;
- list_for_each_entry_safe(req, tmp, &qp_ctx->qp->backlog.list, list) {
+ list_for_each_entry_safe(req, tmp, &qp->backlog.list, list) {
list_del(&req->list);
+ ctx = req->qp_ctx->ctx;
ctx->req_op->buf_unmap(ctx, req);
if (req->req_id >= 0)
sec_free_req_id(req);
@@ -258,9 +260,8 @@ static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp
}
}
-static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
+static void sec_alg_send_backlog(struct hisi_qp *qp)
{
- struct hisi_qp *qp = qp_ctx->qp;
struct sec_req *req, *tmp;
int ret;
@@ -277,7 +278,7 @@ static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
goto unlock;
default:
/* Release memory resources and send all requests through software. */
- sec_alg_send_backlog_soft(ctx, qp_ctx);
+ sec_alg_send_backlog_soft(qp);
goto unlock;
}
}
@@ -306,6 +307,7 @@ static void sec_req_cb(struct hisi_qp *qp, void *resp)
ctx->req_op->buf_unmap(ctx, req);
ctx->req_op->callback(ctx, req, err);
+ sec_alg_send_backlog(qp);
}
static void sec_req_cb3(struct hisi_qp *qp, void *resp)
@@ -331,6 +333,7 @@ static void sec_req_cb3(struct hisi_qp *qp, void *resp)
ctx->req_op->buf_unmap(ctx, req);
ctx->req_op->callback(ctx, req, err);
+ sec_alg_send_backlog(qp);
}
static int sec_alg_send_message_retry(struct sec_req *req)
@@ -1673,8 +1676,6 @@ static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type)
static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req,
int err)
{
- struct sec_qp_ctx *qp_ctx = req->qp_ctx;
-
if (req->req_id >= 0)
sec_free_req_id(req);
@@ -1684,7 +1685,6 @@ static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req,
sec_update_iv(req, SEC_SKCIPHER);
crypto_request_complete(req->base, err);
- sec_alg_send_backlog(ctx, qp_ctx);
}
static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req)
@@ -1923,7 +1923,7 @@ static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err)
struct aead_request *a_req = req->aead_req.aead_req;
struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
size_t authsize = crypto_aead_authsize(tfm);
- struct sec_qp_ctx *qp_ctx = req->qp_ctx;
+ int error = err;
size_t sz;
if (!err && req->c_req.encrypt) {
@@ -1934,15 +1934,14 @@ static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err)
authsize, a_req->cryptlen + a_req->assoclen);
if (unlikely(sz != authsize)) {
dev_err(c->dev, "copy out mac err!\n");
- err = -EINVAL;
+ error = -EINVAL;
}
}
if (req->req_id >= 0)
sec_free_req_id(req);
- crypto_request_complete(req->base, err);
- sec_alg_send_backlog(c, qp_ctx);
+ crypto_request_complete(req->base, error);
}
static void sec_request_uninit(struct sec_req *req)
--
2.43.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 3/3] crypto: hisilicon/hpre - implement full backlog support for hpre driver
2026-05-15 11:45 [PATCH 0/3] crypto: hisilicon - improve backlog handling ZongYu Wu
2026-05-15 11:45 ` [PATCH] crypto: hisilicon/zip - add backlog support for zip ZongYu Wu
2026-05-15 11:46 ` [PATCH 2/3] crypto: hisilicon/sec2 - fix UAF in sec_alg_send_backlog ZongYu Wu
@ 2026-05-15 11:46 ` ZongYu Wu
2 siblings, 0 replies; 5+ messages in thread
From: ZongYu Wu @ 2026-05-15 11:46 UTC (permalink / raw)
To: herbert, davem
Cc: linux-kernel, linux-crypto, fanghao11, liulongfang, qianweili,
wangzhou1
From: lizhi <lizhi206@huawei.com>
When the hardware queue returns -EBUSY, requests are queued instead of
being failed immediately. The driver retries queued requests from the
completion path after earlier requests have finished.
This reduces request failures caused by temporary hardware congestion and
improves throughput and stability under high load.
Signed-off-by: lizhi <lizhi206@huawei.com>
Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com>
---
drivers/crypto/hisilicon/hpre/hpre_crypto.c | 223 +++++++++++++++-----
1 file changed, 166 insertions(+), 57 deletions(-)
diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c
index 09077abbf6ad..5d5c4d5a9fbc 100644
--- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c
+++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c
@@ -30,6 +30,7 @@ struct hpre_ctx;
#define HPRE_DH_G_FLAG 0x02
#define HPRE_TRY_SEND_TIMES 100
#define HPRE_INVLD_REQ_ID (-1)
+#define HPRE_ALG_TYPE_MASK 0x1F
#define HPRE_SQE_ALG_BITS 5
#define HPRE_SQE_DONE_SHIFT 30
@@ -39,6 +40,7 @@ struct hpre_ctx;
#define HPRE_DFX_US_TO_NS 1000
#define HPRE_ENABLE_HPCORE_SHIFT 7
+#define HPRE_ECDH_CLR_DATA_SHIFT 2
/* due to nist p521 */
#define HPRE_ECC_MAX_KSZ 66
@@ -138,6 +140,8 @@ struct hpre_asym_request {
int err;
hpre_cb cb;
struct timespec64 req_time;
+ struct crypto_async_request *base;
+ struct list_head list;
};
static inline unsigned int hpre_align_sz(void)
@@ -241,8 +245,8 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
struct scatterlist *dst,
struct scatterlist *src)
{
- struct device *dev = ctx->dev;
struct hpre_sqe *sqe = &req->req;
+ struct device *dev = ctx->dev;
dma_addr_t tmp;
tmp = le64_to_cpu(sqe->in);
@@ -270,6 +274,34 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
}
}
+static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx,
+ struct hpre_asym_request *req,
+ struct scatterlist *dst,
+ struct scatterlist *src)
+{
+ struct hpre_sqe *sqe = &req->req;
+ struct device *dev = ctx->dev;
+ dma_addr_t dma;
+
+ dma = le64_to_cpu(sqe->in);
+ if (unlikely(dma_mapping_error(dev, dma)))
+ return;
+
+ /* req->src may contain garbage value, check both src and req->src before freeing */
+ if (src && req->src)
+ dma_free_coherent(dev, ctx->key_sz << HPRE_ECDH_CLR_DATA_SHIFT,
+ req->src, dma);
+
+ dma = le64_to_cpu(sqe->out);
+ if (unlikely(dma_mapping_error(dev, dma)))
+ return;
+
+ if (req->dst)
+ dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma);
+ if (dst)
+ dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE);
+}
+
static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,
void **kreq)
{
@@ -323,6 +355,94 @@ static bool hpre_is_bd_timeout(struct hpre_asym_request *req,
return true;
}
+static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg)
+{
+ struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
+ int cnt = 0;
+ int ret;
+
+ do {
+ ret = hisi_qp_send(ctx->qp, msg);
+ if (ret != -EBUSY)
+ break;
+ atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value);
+ } while (cnt++ < HPRE_TRY_SEND_TIMES);
+
+ if (likely(!ret)) {
+ atomic64_inc(&dfx[HPRE_SEND_CNT].value);
+ return ret;
+ }
+
+ if (ret != -EBUSY)
+ atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);
+
+ return ret;
+}
+
+static int hpre_send_backlog(struct hpre_ctx *ctx, struct hpre_sqe *msg)
+{
+ struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
+ int ret;
+
+ ret = hisi_qp_send(ctx->qp, msg);
+ if (likely(!ret))
+ atomic64_inc(&dfx[HPRE_SEND_CNT].value);
+ else if (unlikely(ret != -EBUSY))
+ atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);
+ else
+ atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value);
+
+ return ret;
+}
+
+static void hpre_alg_hw_data_clr_all(struct hpre_ctx *ctx, struct hpre_asym_request *h_req)
+{
+ switch (le32_to_cpu(h_req->req.dw0) & HPRE_ALG_TYPE_MASK) {
+ case HPRE_ALG_DH_G2:
+ case HPRE_ALG_DH:
+ hpre_hw_data_clr_all(ctx, h_req, h_req->areq.dh->dst, h_req->areq.dh->src);
+ break;
+ case HPRE_ALG_NC_NCRT:
+ case HPRE_ALG_NC_CRT:
+ hpre_hw_data_clr_all(ctx, h_req, h_req->areq.rsa->dst, h_req->areq.rsa->src);
+ break;
+ case HPRE_ALG_ECC_MUL:
+ hpre_ecdh_hw_data_clr_all(ctx, h_req, h_req->areq.ecdh->dst, h_req->areq.ecdh->src);
+ break;
+ default:
+ break;
+ }
+}
+
+static void hpre_alg_send_backlog(struct hisi_qp *qp)
+{
+ struct hpre_asym_request *req, *tmp;
+ int ret;
+
+ spin_lock_bh(&qp->backlog.lock);
+ list_for_each_entry_safe(req, tmp, &qp->backlog.list, list) {
+ ret = hpre_send_backlog(req->ctx, &req->req);
+ switch (ret) {
+ case 0:
+ list_del(&req->list);
+ crypto_request_complete(req->base, -EINPROGRESS);
+ break;
+ case -EBUSY:
+ /* Device is busy and stop send any request. */
+ goto unlock;
+ default:
+ /* Current no fallback for any send error. */
+ list_del(&req->list);
+ hpre_alg_hw_data_clr_all(req->ctx, req);
+ crypto_request_complete(req->base, -EIO);
+ break;
+ }
+ }
+
+unlock:
+ spin_unlock_bh(&qp->backlog.lock);
+}
+
static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp)
{
struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
@@ -377,6 +497,7 @@ static void hpre_alg_cb(struct hisi_qp *qp, void *resp)
}
h_req->cb(h_req->ctx, resp);
+ hpre_alg_send_backlog(qp);
}
static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type)
@@ -450,25 +571,39 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
return 0;
}
-static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg)
+static int hpre_alg_try_enqueue(struct hpre_asym_request *hpre_req)
{
- struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
- int ctr = 0;
+ struct hisi_qp *qp = hpre_req->ctx->qp;
+
+ /* Check if any request is already backlogged */
+ if (!list_empty(&qp->backlog.list))
+ return -EBUSY;
+
+ /* Try to enqueue to HW ring */
+ return hpre_send_backlog(hpre_req->ctx, &hpre_req->req);
+}
+
+static int hpre_alg_send_message(struct hpre_asym_request *hpre_req)
+{
+ struct hisi_qp *qp = hpre_req->ctx->qp;
int ret;
- do {
- atomic64_inc(&dfx[HPRE_SEND_CNT].value);
- ret = hisi_qp_send(ctx->qp, msg);
- if (ret != -EBUSY)
- break;
- atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value);
- } while (ctr++ < HPRE_TRY_SEND_TIMES);
+ if (!(hpre_req->base->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
+ ret = hpre_send(hpre_req->ctx, &hpre_req->req);
+ if (ret == -EBUSY)
+ return -ENOSPC;
+ } else {
+ ret = hpre_alg_try_enqueue(hpre_req);
+ if (ret == -EBUSY) {
+ spin_lock_bh(&qp->backlog.lock);
+ list_add_tail(&hpre_req->list, &qp->backlog.list);
+ spin_unlock_bh(&qp->backlog.lock);
+ return -EBUSY;
+ }
+ }
if (likely(!ret))
- return ret;
-
- if (ret != -EBUSY)
- atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);
+ return -EINPROGRESS;
return ret;
}
@@ -482,6 +617,7 @@ static int hpre_dh_compute_value(struct kpp_request *req)
struct hpre_sqe *msg = &hpre_req->req;
int ret;
+ hpre_req->base = &req->base;
ret = hpre_msg_request_set(ctx, req, false);
if (unlikely(ret))
return ret;
@@ -503,14 +639,12 @@ static int hpre_dh_compute_value(struct kpp_request *req)
else
msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH);
- /* success */
- ret = hpre_send(ctx, msg);
- if (likely(!ret))
- return -EINPROGRESS;
+ ret = hpre_alg_send_message(hpre_req);
+ if (likely(ret == -EINPROGRESS || ret == -EBUSY))
+ return ret;
clear_all:
hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
-
return ret;
}
@@ -755,6 +889,7 @@ static int hpre_rsa_enc(struct akcipher_request *req)
struct hpre_sqe *msg = &hpre_req->req;
int ret;
+ hpre_req->base = &req->base;
/* For unsupported key size and unavailable devices, use soft tfm instead */
if (ctx->fallback) {
akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
@@ -781,10 +916,9 @@ static int hpre_rsa_enc(struct akcipher_request *req)
if (unlikely(ret))
goto clear_all;
- /* success */
- ret = hpre_send(ctx, msg);
- if (likely(!ret))
- return -EINPROGRESS;
+ ret = hpre_alg_send_message(hpre_req);
+ if (likely(ret == -EINPROGRESS || ret == -EBUSY))
+ return ret;
clear_all:
hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
@@ -801,6 +935,7 @@ static int hpre_rsa_dec(struct akcipher_request *req)
struct hpre_sqe *msg = &hpre_req->req;
int ret;
+ hpre_req->base = &req->base;
/* For unsupported key size and unavailable devices, use soft tfm instead */
if (ctx->fallback) {
akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
@@ -834,10 +969,9 @@ static int hpre_rsa_dec(struct akcipher_request *req)
if (unlikely(ret))
goto clear_all;
- /* success */
- ret = hpre_send(ctx, msg);
- if (likely(!ret))
- return -EINPROGRESS;
+ ret = hpre_alg_send_message(hpre_req);
+ if (likely(ret == -EINPROGRESS || ret == -EBUSY))
+ return ret;
clear_all:
hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
@@ -1387,32 +1521,6 @@ static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
return 0;
}
-static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx,
- struct hpre_asym_request *req,
- struct scatterlist *dst,
- struct scatterlist *src)
-{
- struct device *dev = ctx->dev;
- struct hpre_sqe *sqe = &req->req;
- dma_addr_t dma;
-
- dma = le64_to_cpu(sqe->in);
- if (unlikely(dma_mapping_error(dev, dma)))
- return;
-
- if (src && req->src)
- dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma);
-
- dma = le64_to_cpu(sqe->out);
- if (unlikely(dma_mapping_error(dev, dma)))
- return;
-
- if (req->dst)
- dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma);
- if (dst)
- dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE);
-}
-
static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp)
{
unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
@@ -1538,6 +1646,7 @@ static int hpre_ecdh_compute_value(struct kpp_request *req)
struct hpre_sqe *msg = &hpre_req->req;
int ret;
+ hpre_req->base = &req->base;
ret = hpre_ecdh_msg_request_set(ctx, req);
if (unlikely(ret)) {
dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret);
@@ -1563,9 +1672,9 @@ static int hpre_ecdh_compute_value(struct kpp_request *req)
msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL);
msg->resv1 = ctx->enable_hpcore << HPRE_ENABLE_HPCORE_SHIFT;
- ret = hpre_send(ctx, msg);
- if (likely(!ret))
- return -EINPROGRESS;
+ ret = hpre_alg_send_message(hpre_req);
+ if (likely(ret == -EINPROGRESS || ret == -EBUSY))
+ return ret;
clear_all:
hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
--
2.43.0
^ permalink raw reply related [flat|nested] 5+ messages in thread