From: Leon Romanovsky <leon@kernel.org>
To: David Hu <dhu@x6u.co>
Cc: Tatyana Nikolova <tatyana.e.nikolova@intel.com>,
Jason Gunthorpe <jgg@ziepe.ca>,
linux-rdma@vger.kernel.org, linux-kernel@vger.kernel.org,
jmoroni@google.com, eagleshaw@google.com, xuehaohu@google.com
Subject: Re: [PATCH rdma-next v2 4/6] RDMA/irdma: Split CQ setup to umode and kmode
Date: Thu, 15 Oct 2026 21:17:13 +0300 [thread overview]
Message-ID: <20261015181713.GB219304@unreal> (raw)
In-Reply-To: <20261015011940.4122452-5-dhu@x6u.co>
On Thu, Oct 15, 2026 at 01:19:38AM +0000, David Hu wrote:
> From: David Hu <xuehaohu@google.com>
>
> This split is similar to QP, and SRQ that is already present in existing
> irdma driver. Only variable plumbing was done, while umode and kmode setup
> logic is identical as before.
>
> Signed-off-by: David Hu <xuehaohu@google.com>
> ---
> drivers/infiniband/hw/irdma/verbs.c | 232 +++++++++++++++-------------
> 1 file changed, 126 insertions(+), 106 deletions(-)
>
> diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
> index ab25445f3828..7162b37fa6ff 100644
> --- a/drivers/infiniband/hw/irdma/verbs.c
> +++ b/drivers/infiniband/hw/irdma/verbs.c
> @@ -2624,6 +2624,126 @@ static inline int cq_validate_flags(u32 flags, u8 hw_rev)
> return flags & ~IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION ? -EOPNOTSUPP : 0;
> }
>
> +static int irdma_setup_kmode_cq(struct irdma_device *iwdev,
> + struct irdma_cq *iwcq,
> + struct irdma_cq_init_info *info,
> + const struct ib_cq_init_attr *attr)
> +{
> + struct irdma_cq_uk_init_info *ukinfo = &info->cq_uk_init_info;
> + struct irdma_pci_f *rf = iwdev->rf;
> + struct irdma_sc_dev *dev = &rf->sc_dev;
> + int entries = attr->cqe;
> + bool cqe_64byte_ena;
> + u8 cqe_size;
> + int rsize;
> +
> + if (entries < 1 || entries > rf->max_cqe)
> + return -EINVAL;
This if() is impossible.
> +
> + cqe_64byte_ena = dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_64_BYTE_CQE ?
> + true : false;
> + cqe_size = cqe_64byte_ena ? 64 : 32;
> +
> + entries += 2;
> + if (!cqe_64byte_ena && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
> + entries *= 2;
> +
> + if (entries & 1)
> + entries += 1; /* cq size must be an even number */
> +
> + if (entries * cqe_size == IRDMA_HW_PAGE_SIZE)
> + entries += 2;
> +
> + ukinfo->cq_size = entries;
> +
> + if (cqe_64byte_ena)
> + rsize = ukinfo->cq_size * sizeof(struct irdma_extended_cqe);
> + else
> + rsize = ukinfo->cq_size * sizeof(struct irdma_cqe);
> + iwcq->kmem.size = ALIGN(round_up(rsize, 256), 256);
> + iwcq->kmem.va = dma_alloc_coherent(dev->hw->device,
> + iwcq->kmem.size,
> + &iwcq->kmem.pa, GFP_KERNEL);
> + if (!iwcq->kmem.va)
> + return -ENOMEM;
> +
> + iwcq->kmem_shadow.size = ALIGN(IRDMA_SHADOW_AREA_SIZE << 3, 64);
> + iwcq->kmem_shadow.va = dma_alloc_coherent(dev->hw->device,
> + iwcq->kmem_shadow.size,
> + &iwcq->kmem_shadow.pa,
> + GFP_KERNEL);
> + if (!iwcq->kmem_shadow.va)
> + return -ENOMEM;
> +
> + info->shadow_area_pa = iwcq->kmem_shadow.pa;
> + ukinfo->shadow_area = iwcq->kmem_shadow.va;
> + ukinfo->cq_base = iwcq->kmem.va;
> + info->cq_base_pa = iwcq->kmem.pa;
> +
> + return 0;
> +}
> +
> +static int irdma_setup_umode_cq(struct irdma_device *iwdev,
> + struct irdma_cq *iwcq,
> + struct irdma_cq_init_info *info,
> + struct ib_udata *udata)
> +{
> + struct irdma_create_cq_req req = {};
> + struct irdma_pci_f *rf = iwdev->rf;
> + struct irdma_ucontext *ucontext;
> + struct irdma_cq_mr *cqmr_shadow;
> + struct irdma_cq_mr *cqmr;
> + unsigned long flags;
> + int err_code;
> +
> + iwcq->user_mode = true;
> + ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
> + ibucontext);
> + /* Even though the last member of struct irdma_create_cq_req
> + * was always user_shadow_area, we need backwards compat with
> + * the legacy i40iw struct i40iw_ucreate_cq which stopped
> + * at user_cq_buffer.
> + */
> + err_code = ib_copy_validate_udata_in(udata, req, user_cq_buf);
> + if (err_code)
> + return err_code;
> +
> + spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
> + iwcq->iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf,
> + &ucontext->cq_reg_mem_list);
> + spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
> + if (!iwcq->iwpbl)
> + return -EPROTO;
> +
> + cqmr = &iwcq->iwpbl->cq_mr;
> +
> + if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags &
> + IRDMA_FEATURE_CQ_RESIZE) {
> + spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
> + iwcq->iwpbl_shadow =
> + irdma_get_pbl((unsigned long)req.user_shadow_area,
> + &ucontext->cq_reg_mem_list);
> + spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
> +
> + if (!iwcq->iwpbl_shadow)
> + return -EPROTO;
> +
> + cqmr_shadow = &iwcq->iwpbl_shadow->cq_mr;
> + info->shadow_area_pa = cqmr_shadow->cq_pbl.addr;
> + } else {
> + info->shadow_area_pa = cqmr->shadow;
> + }
> + if (iwcq->iwpbl->pbl_allocated) {
> + info->virtual_map = true;
> + info->pbl_chunk_size = 1;
> + info->first_pm_pbl_idx = cqmr->cq_pbl.idx;
> + } else {
> + info->cq_base_pa = cqmr->cq_pbl.addr;
> + }
> +
> + return 0;
> +}
> +
> /**
> * irdma_create_cq - create cq
> * @ibcq: CQ allocated
> @@ -2647,11 +2767,9 @@ static int irdma_create_cq(struct ib_cq *ibcq,
> struct irdma_cqp_request *cqp_request;
> struct cqp_cmds_info *cqp_info;
> struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info;
> - unsigned long flags;
> int err_code;
> int entries = attr->cqe;
> bool cqe_64byte_ena;
> - u8 cqe_size;
>
> err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev);
> if (err_code)
> @@ -2679,7 +2797,6 @@ static int irdma_create_cq(struct ib_cq *ibcq,
> ukinfo->cq_id = cq_num;
> cqe_64byte_ena = dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_64_BYTE_CQE ?
> true : false;
> - cqe_size = cqe_64byte_ena ? 64 : 32;
> ukinfo->avoid_mem_cflct = cqe_64byte_ena;
> iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size;
> if (attr->comp_vector < rf->ceqs_count)
> @@ -2689,109 +2806,12 @@ static int irdma_create_cq(struct ib_cq *ibcq,
> info.type = IRDMA_CQ_TYPE_IWARP;
> info.vsi = &iwdev->vsi;
>
> - if (udata) {
> - struct irdma_ucontext *ucontext;
> - struct irdma_create_cq_req req = {};
> - struct irdma_cq_mr *cqmr;
> - struct irdma_cq_mr *cqmr_shadow;
> -
> - iwcq->user_mode = true;
> - ucontext =
> - rdma_udata_to_drv_context(udata, struct irdma_ucontext,
> - ibucontext);
> - /* Even though the last member of struct irdma_create_cq_req
> - * was always user_shadow_area, we need backwards compat with
> - * the legacy i40iw struct i40iw_ucreate_cq which stopped
> - * at user_cq_buffer.
> - */
> - err_code = ib_copy_validate_udata_in(udata, req, user_cq_buf);
> - if (err_code)
> - goto cq_free_rsrc;
> -
> - spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
> - iwcq->iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf,
> - &ucontext->cq_reg_mem_list);
> - spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
> - if (!iwcq->iwpbl) {
> - err_code = -EPROTO;
> - goto cq_free_rsrc;
> - }
> -
> - cqmr = &iwcq->iwpbl->cq_mr;
> -
> - if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags &
> - IRDMA_FEATURE_CQ_RESIZE) {
> - spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
> - iwcq->iwpbl_shadow = irdma_get_pbl(
> - (unsigned long)req.user_shadow_area,
> - &ucontext->cq_reg_mem_list);
> - spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
> -
> - if (!iwcq->iwpbl_shadow) {
> - err_code = -EPROTO;
> - goto cq_free_rsrc;
> - }
> - cqmr_shadow = &iwcq->iwpbl_shadow->cq_mr;
> - info.shadow_area_pa = cqmr_shadow->cq_pbl.addr;
> - } else {
> - info.shadow_area_pa = cqmr->shadow;
> - }
> - if (iwcq->iwpbl->pbl_allocated) {
> - info.virtual_map = true;
> - info.pbl_chunk_size = 1;
> - info.first_pm_pbl_idx = cqmr->cq_pbl.idx;
> - } else {
> - info.cq_base_pa = cqmr->cq_pbl.addr;
> - }
> - } else {
> - /* Kmode allocations */
> - int rsize;
> -
> - if (entries < 1 || entries > rf->max_cqe) {
> - err_code = -EINVAL;
> - goto cq_free_rsrc;
> - }
> -
> - entries += 2;
> - if (!cqe_64byte_ena && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
> - entries *= 2;
> -
> - if (entries & 1)
> - entries += 1; /* cq size must be an even number */
> -
> - if (entries * cqe_size == IRDMA_HW_PAGE_SIZE)
> - entries += 2;
> -
> - ukinfo->cq_size = entries;
> -
> - if (cqe_64byte_ena)
> - rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_extended_cqe);
> - else
> - rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe);
> - iwcq->kmem.size = ALIGN(round_up(rsize, 256), 256);
> - iwcq->kmem.va = dma_alloc_coherent(dev->hw->device,
> - iwcq->kmem.size,
> - &iwcq->kmem.pa, GFP_KERNEL);
> - if (!iwcq->kmem.va) {
> - err_code = -ENOMEM;
> - goto cq_free_rsrc;
> - }
> -
> - iwcq->kmem_shadow.size = ALIGN(IRDMA_SHADOW_AREA_SIZE << 3,
> - 64);
> - iwcq->kmem_shadow.va = dma_alloc_coherent(dev->hw->device,
> - iwcq->kmem_shadow.size,
> - &iwcq->kmem_shadow.pa,
> - GFP_KERNEL);
> - if (!iwcq->kmem_shadow.va) {
> - err_code = -ENOMEM;
> - goto cq_free_rsrc;
> - }
> - info.shadow_area_pa = iwcq->kmem_shadow.pa;
> - ukinfo->shadow_area = iwcq->kmem_shadow.va;
> - ukinfo->cq_base = iwcq->kmem.va;
> - info.cq_base_pa = iwcq->kmem.pa;
> - }
> + if (udata)
> + err_code = irdma_setup_umode_cq(iwdev, iwcq, &info, udata);
> + else
> + err_code = irdma_setup_kmode_cq(iwdev, iwcq, &info, attr);
No, the idea was to actually implement and set ,crate_user_cq() callback
and clean as much as possible from existing .create_cq() callback.
Thanks
> + if (err_code)
> + goto cq_free_rsrc;
>
> info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2,
> (u32)IRDMA_MAX_CQ_READ_THRESH);
> --
> 2.56.0.385.gd3acb90ef8-goog
>
>
next prev parent reply other threads:[~2026-10-15 18:17 UTC|newest]
Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-15 1:19 [PATCH rdma-next v2 0/6] RDMA/irdma: Add support for ioctl David Hu
2026-10-15 1:19 ` [PATCH rdma-next v2 1/6] RDMA/irdma: Plumb uverbs_attr_bundle for QP, SRQ David Hu
2026-10-15 1:31 ` sashiko-bot
2026-10-15 1:19 ` [PATCH rdma-next v2 2/6] RDMA/irdma: Register irdma UVERBS ioctl namespace David Hu
2026-10-15 1:33 ` sashiko-bot
2026-10-15 1:19 ` [PATCH rdma-next v2 3/6] RDMA/irdma: Support UMEM attribute for QP creation David Hu
2026-10-15 1:36 ` sashiko-bot
2026-10-15 1:19 ` [PATCH rdma-next v2 4/6] RDMA/irdma: Split CQ setup to umode and kmode David Hu
2026-10-15 1:30 ` sashiko-bot
2026-10-15 18:17 ` Leon Romanovsky [this message]
2026-10-15 21:19 ` David Hu
2026-10-15 1:19 ` [PATCH rdma-next v2 5/6] RDMA/irdma: Support UMEM attribute for CQ creation David Hu
2026-10-15 1:33 ` sashiko-bot
2026-10-15 1:19 ` [PATCH rdma-next v2 6/6] RDMA/irdma: Support UMEM attribute for SRQ creation David Hu
2026-10-15 1:32 ` sashiko-bot
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261015181713.GB219304@unreal \
--to=leon@kernel.org \
--cc=dhu@x6u.co \
--cc=eagleshaw@google.com \
--cc=jgg@ziepe.ca \
--cc=jmoroni@google.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=tatyana.e.nikolova@intel.com \
--cc=xuehaohu@google.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.