From: Christoph Hellwig <hch-jcswGhMUV9g@public.gmane.org>
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
Cc: sagi-NQWnxTmZq1alnMjI0IkVqw@public.gmane.org
Subject: [PATCH 3/6] IB/isert: use implicit CQ allocation
Date: Fri, 9 Sep 2016 14:36:24 +0200 [thread overview]
Message-ID: <1473424587-13818-4-git-send-email-hch@lst.de> (raw)
In-Reply-To: <1473424587-13818-1-git-send-email-hch-jcswGhMUV9g@public.gmane.org>
From: Sagi Grimberg <sagi-NQWnxTmZq1alnMjI0IkVqw@public.gmane.org>
Signed-off-by: Sagi Grimberg <sagi-NQWnxTmZq1alnMjI0IkVqw@public.gmane.org>
[hch: ported to the new API]
Signed-off-by: Christoph Hellwig <hch-jcswGhMUV9g@public.gmane.org>
---
drivers/infiniband/ulp/isert/ib_isert.c | 164 ++++----------------------------
drivers/infiniband/ulp/isert/ib_isert.h | 16 ----
2 files changed, 18 insertions(+), 162 deletions(-)
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index 8df608e..78f6b2a 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -35,8 +35,6 @@
#define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
#define ISER_MAX_TX_CQ_LEN \
((ISERT_QP_MAX_REQ_DTOS + ISCSI_DEF_XMIT_CMDS_MAX) * ISERT_MAX_CONN)
-#define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \
- ISERT_MAX_CONN)
static int isert_debug_level;
module_param_named(debug_level, isert_debug_level, int, 0644);
@@ -89,55 +87,26 @@ isert_qp_event_callback(struct ib_event *e, void *context)
}
}
-static struct isert_comp *
-isert_comp_get(struct isert_conn *isert_conn)
-{
- struct isert_device *device = isert_conn->device;
- struct isert_comp *comp;
- int i, min = 0;
-
- mutex_lock(&device_list_mutex);
- for (i = 0; i < device->comps_used; i++)
- if (device->comps[i].active_qps <
- device->comps[min].active_qps)
- min = i;
- comp = &device->comps[min];
- comp->active_qps++;
- mutex_unlock(&device_list_mutex);
-
- isert_info("conn %p, using comp %p min_index: %d\n",
- isert_conn, comp, min);
-
- return comp;
-}
-
-static void
-isert_comp_put(struct isert_comp *comp)
-{
- mutex_lock(&device_list_mutex);
- comp->active_qps--;
- mutex_unlock(&device_list_mutex);
-}
-
static struct ib_qp *
-isert_create_qp(struct isert_conn *isert_conn,
- struct isert_comp *comp,
- struct rdma_cm_id *cma_id)
+isert_create_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
{
struct isert_device *device = isert_conn->device;
struct ib_qp_init_attr attr;
int ret;
- memset(&attr, 0, sizeof(struct ib_qp_init_attr));
+ memset(&attr, 0, sizeof(attr));
+ attr.create_flags = IB_QP_CREATE_ASSIGN_CQS;
attr.event_handler = isert_qp_event_callback;
attr.qp_context = isert_conn;
- attr.send_cq = comp->cq;
- attr.recv_cq = comp->cq;
+ attr.poll_ctx = IB_POLL_WORKQUEUE;
+
attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1;
- attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX;
attr.cap.max_send_sge = device->ib_device->attrs.max_sge;
+
+ attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
attr.cap.max_recv_sge = 1;
+
attr.sq_sig_type = IB_SIGNAL_REQ_WR;
attr.qp_type = IB_QPT_RC;
if (device->pi_capable)
@@ -153,25 +122,6 @@ isert_create_qp(struct isert_conn *isert_conn,
}
static int
-isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
-{
- struct isert_comp *comp;
- int ret;
-
- comp = isert_comp_get(isert_conn);
- isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id);
- if (IS_ERR(isert_conn->qp)) {
- ret = PTR_ERR(isert_conn->qp);
- goto err;
- }
-
- return 0;
-err:
- isert_comp_put(comp);
- return ret;
-}
-
-static int
isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
{
struct isert_device *device = isert_conn->device;
@@ -239,63 +189,6 @@ isert_free_rx_descriptors(struct isert_conn *isert_conn)
isert_conn->rx_descs = NULL;
}
-static void
-isert_free_comps(struct isert_device *device)
-{
- int i;
-
- for (i = 0; i < device->comps_used; i++) {
- struct isert_comp *comp = &device->comps[i];
-
- if (comp->cq)
- ib_free_cq(comp->cq);
- }
- kfree(device->comps);
-}
-
-static int
-isert_alloc_comps(struct isert_device *device)
-{
- int i, max_cqe, ret = 0;
-
- device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(),
- device->ib_device->num_comp_vectors));
-
- isert_info("Using %d CQs, %s supports %d vectors support "
- "pi_capable %d\n",
- device->comps_used, device->ib_device->name,
- device->ib_device->num_comp_vectors,
- device->pi_capable);
-
- device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp),
- GFP_KERNEL);
- if (!device->comps) {
- isert_err("Unable to allocate completion contexts\n");
- return -ENOMEM;
- }
-
- max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe);
-
- for (i = 0; i < device->comps_used; i++) {
- struct isert_comp *comp = &device->comps[i];
-
- comp->device = device;
- comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i,
- IB_POLL_WORKQUEUE);
- if (IS_ERR(comp->cq)) {
- isert_err("Unable to allocate cq\n");
- ret = PTR_ERR(comp->cq);
- comp->cq = NULL;
- goto out_cq;
- }
- }
-
- return 0;
-out_cq:
- isert_free_comps(device);
- return ret;
-}
-
static int
isert_create_device_ib_res(struct isert_device *device)
{
@@ -305,16 +198,11 @@ isert_create_device_ib_res(struct isert_device *device)
isert_dbg("devattr->max_sge: %d\n", ib_dev->attrs.max_sge);
isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd);
- ret = isert_alloc_comps(device);
- if (ret)
- goto out;
-
device->pd = ib_alloc_pd(ib_dev, 0);
if (IS_ERR(device->pd)) {
- ret = PTR_ERR(device->pd);
isert_err("failed to allocate pd, device %p, ret=%d\n",
device, ret);
- goto out_cq;
+ return PTR_ERR(device->pd);
}
/* Check signature cap */
@@ -322,22 +210,6 @@ isert_create_device_ib_res(struct isert_device *device)
IB_DEVICE_SIGNATURE_HANDOVER ? true : false;
return 0;
-
-out_cq:
- isert_free_comps(device);
-out:
- if (ret > 0)
- ret = -EINVAL;
- return ret;
-}
-
-static void
-isert_free_device_ib_res(struct isert_device *device)
-{
- isert_info("device %p\n", device);
-
- ib_dealloc_pd(device->pd);
- isert_free_comps(device);
}
static void
@@ -347,7 +219,7 @@ isert_device_put(struct isert_device *device)
device->refcount--;
isert_info("device %p refcount %d\n", device, device->refcount);
if (!device->refcount) {
- isert_free_device_ib_res(device);
+ ib_dealloc_pd(device->pd);
list_del(&device->dev_node);
kfree(device);
}
@@ -540,13 +412,15 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
isert_set_nego_params(isert_conn, &event->param.conn);
- ret = isert_conn_setup_qp(isert_conn, cma_id);
- if (ret)
+ isert_conn->qp = isert_create_qp(isert_conn, cma_id);
+ if (IS_ERR(isert_conn->qp)) {
+ ret = PTR_ERR(isert_conn->qp);
goto out_conn_dev;
+ }
ret = isert_login_post_recv(isert_conn);
if (ret)
- goto out_conn_dev;
+ goto out_conn_qp;
ret = isert_rdma_accept(isert_conn);
if (ret)
@@ -558,6 +432,8 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
return 0;
+out_conn_qp:
+ ib_destroy_qp(isert_conn->qp);
out_conn_dev:
isert_device_put(device);
out_rsp_dma_map:
@@ -581,12 +457,8 @@ isert_connect_release(struct isert_conn *isert_conn)
if (isert_conn->cm_id)
rdma_destroy_id(isert_conn->cm_id);
- if (isert_conn->qp) {
- struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context;
-
- isert_comp_put(comp);
+ if (isert_conn->qp)
ib_destroy_qp(isert_conn->qp);
- }
if (isert_conn->login_req_buf)
isert_free_login_buf(isert_conn);
diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h
index fc791ef..2e85eed 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.h
+++ b/drivers/infiniband/ulp/isert/ib_isert.h
@@ -162,27 +162,11 @@ struct isert_conn {
#define ISERT_MAX_CQ 64
-/**
- * struct isert_comp - iSER completion context
- *
- * @device: pointer to device handle
- * @cq: completion queue
- * @active_qps: Number of active QPs attached
- * to completion context
- */
-struct isert_comp {
- struct isert_device *device;
- struct ib_cq *cq;
- int active_qps;
-};
-
struct isert_device {
bool pi_capable;
int refcount;
struct ib_device *ib_device;
struct ib_pd *pd;
- struct isert_comp *comps;
- int comps_used;
struct list_head dev_node;
};
--
2.1.4
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
next prev parent reply other threads:[~2016-09-09 12:36 UTC|newest]
Thread overview: 17+ messages / expand[flat|nested] mbox.gz Atom feed top
2016-09-09 12:36 RFC: CQ pools and implicit CQ resource allocation Christoph Hellwig
[not found] ` <1473424587-13818-1-git-send-email-hch-jcswGhMUV9g@public.gmane.org>
2016-09-09 12:36 ` [PATCH 1/6] IB/core: add implicit CQ allocation Christoph Hellwig
2016-09-09 12:36 ` [PATCH 2/6] nvmet-rdma: use " Christoph Hellwig
2016-09-09 12:36 ` Christoph Hellwig [this message]
2016-09-09 12:36 ` [PATCH 4/6] IB/srpt: " Christoph Hellwig
2016-09-09 12:36 ` [PATCH 5/6] IB/iser: " Christoph Hellwig
2016-09-09 12:36 ` [PATCH 6/6] nvme-rdma: " Christoph Hellwig
2016-09-11 6:39 ` RFC: CQ pools and implicit CQ resource allocation Sagi Grimberg
2016-09-11 6:44 ` Sagi Grimberg
[not found] ` <d0c645eb-3674-2841-bdb7-8b9e6fd46473-NQWnxTmZq1alnMjI0IkVqw@public.gmane.org>
2016-09-12 14:42 ` Steve Wise
2016-09-12 20:07 ` Sagi Grimberg
[not found] ` <6dd3ecbd-7109-95ff-9c86-dfea9e515538-NQWnxTmZq1alnMjI0IkVqw@public.gmane.org>
2016-09-12 20:22 ` Steve Wise
2016-09-12 21:12 ` Sagi Grimberg
2016-09-14 3:29 ` Bart Van Assche
2016-09-12 15:03 ` Chuck Lever
[not found] ` <BCF7E723-FC50-4497-9E0B-1157CF2D4185-QHcLZuEGTsvQT0dZR+AlfA@public.gmane.org>
2016-09-12 20:25 ` Sagi Grimberg
2016-09-16 6:02 ` Bart Van Assche
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1473424587-13818-4-git-send-email-hch@lst.de \
--to=hch-jcswghmuv9g@public.gmane.org \
--cc=linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org \
--cc=sagi-NQWnxTmZq1alnMjI0IkVqw@public.gmane.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox