RE: [PATCH RFC 2/2] RDMA/isert: Support iWARP transport

["Steve Wise" <swise@opengridcomputing.com>]

> -----Original Message-----
> From: linux-rdma-owner@vger.kernel.org [mailto:linux-rdma-owner@vger.kernel.org] On Behalf Of Sagi Grimberg
> Sent: Thursday, June 25, 2015 11:51 AM
> To: Steve Wise; linux-rdma@vger.kernel.org
> Cc: Or Gerlitz; Roi Dayan; target-devel
> Subject: Re: [PATCH RFC 2/2] RDMA/isert: Support iWARP transport
> 
> On 6/25/2015 6:39 PM, Steve Wise wrote:
> > Memory regions that are the target of an iWARP RDMA READ RESPONSE need
> > REMOTE_WRITE access rights.  So enable REMOTE_WRITE for iWARP devices.
> >
> > iWARP RDMA READ target sge depth is 1.  So save the max_read_sge in
> > the target device structure and use that when creating RDMA_READ work
> > requests.
> 
> Hi Steve,
> 
> CC'ing target-devel...
> 
> >
> > Signed-off-by: Steve Wise <swise@opengridcomputing.com>
> > Tested-by: Vasu Dev <vasu.dev@intel.com>
> > ---
> >   drivers/infiniband/ulp/isert/ib_isert.c |   55 ++++++++++++++++++++++++++-----
> >   drivers/infiniband/ulp/isert/ib_isert.h |    1 +
> >   2 files changed, 48 insertions(+), 8 deletions(-)
> >
> > diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
> > index 9e7b492..b5cde5d 100644
> > --- a/drivers/infiniband/ulp/isert/ib_isert.c
> > +++ b/drivers/infiniband/ulp/isert/ib_isert.c
> > @@ -165,6 +165,11 @@ isert_create_qp(struct isert_conn *isert_conn,
> >   	attr.cap.max_send_sge = max(2, device->dev_attr.max_sge - 2);
> >   	isert_conn->max_sge = attr.cap.max_send_sge;
> >
> > +	if (rdma_cap_read_multi_sge(device->ib_device, cma_id->port_num))
> > +		isert_conn->max_read_sge = isert_conn->max_sge;
> > +	else
> > +		isert_conn->max_read_sge = 1;
> > +
> 
> I think it would make sense to change now max_sge to max_write_sge.
> 

Ok.

> And, shouldn't we just use:
> max_wr_sge = max(2, device->dev_attr.max_sge - 2);
> max_rd_sge = device->dev_attr.max_sge_rd;
> 
> Does the Chelsio device reports max_sge_rd != 1 ?
> 

Yes, but I wasn't sure max_sge_rd is really the read target max sge.  I don't see it being set by the mlx* drivers.  Also since we have the new rdma_cap_read_multi_sge() helper, I thought I should use it. :)

> 
> >   	attr.cap.max_recv_sge = 1;
> >   	attr.sq_sig_type = IB_SIGNAL_REQ_WR;
> >   	attr.qp_type = IB_QPT_RC;
> > @@ -348,6 +353,17 @@ out_cq:
> >   	return ret;
> >   }
> >
> > +static int any_port_is_iwarp(struct isert_device *device)
> > +{
> > +	int i;
> > +
> > +	for (i = rdma_start_port(device->ib_device);
> > +	     i <= rdma_end_port(device->ib_device); i++)
> > +		if (rdma_protocol_iwarp(device->ib_device, i))
> > +			return 1;
> > +	return 0;
> > +}
> > +
> 
> This does not look like it belongs here... Can we
> place this in rdma_cm?
>

Ok...that makes sense.
 
> >   static int
> >   isert_create_device_ib_res(struct isert_device *device)
> >   {
> > @@ -383,7 +399,17 @@ isert_create_device_ib_res(struct isert_device *device)
> >   		goto out_cq;
> >   	}
> >
> > -	device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE);
> > +	/*
> > +	 * IWARP transports need REMOTE_WRITE for MRs used as the target of
> > +	 * an RDMA_READ.  Since the DMA MR is used for all ports, then if
> > +	 * any port is running IWARP, add REMOTE_WRITE.
> > +	 */
> > +	if (any_port_is_iwarp(device))
> > +		device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE |
> > +						       IB_ACCESS_REMOTE_WRITE);
> > +	else
> > +		device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE);
> > +
> >   	if (IS_ERR(device->mr)) {
> >   		ret = PTR_ERR(device->mr);
> >   		isert_err("failed to create dma mr, device %p, ret=%d\n",
> > @@ -2375,7 +2401,7 @@ isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
> >   static int
> >   isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
> >   		    struct ib_sge *ib_sge, struct ib_send_wr *send_wr,
> > -		    u32 data_left, u32 offset)
> > +		    u32 data_left, u32 offset, u32 max_sge)
> >   {
> >   	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
> >   	struct scatterlist *sg_start, *tmp_sg;
> > @@ -2386,7 +2412,7 @@ isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
> >
> >   	sg_off = offset / PAGE_SIZE;
> >   	sg_start = &cmd->se_cmd.t_data_sg[sg_off];
> > -	sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge);
> > +	sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, max_sge);
> >   	page_off = offset % PAGE_SIZE;
> >
> >   	send_wr->sg_list = ib_sge;
> > @@ -2430,8 +2456,9 @@ isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
> >   	struct isert_data_buf *data = &wr->data;
> >   	struct ib_send_wr *send_wr;
> >   	struct ib_sge *ib_sge;
> > -	u32 offset, data_len, data_left, rdma_write_max, va_offset = 0;
> > +	u32 offset, data_len, data_left, rdma_max_len, va_offset = 0;
> >   	int ret = 0, i, ib_sge_cnt;
> > +	u32 max_sge;
> >
> >   	isert_cmd->tx_desc.isert_cmd = isert_cmd;
> >
> > @@ -2453,7 +2480,12 @@ isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
> >   	}
> >   	wr->ib_sge = ib_sge;
> >
> > -	wr->send_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge);
> > +	if (wr->iser_ib_op == ISER_IB_RDMA_WRITE)
> > +		max_sge = isert_conn->max_sge;
> > +	else
> > +		max_sge =  isert_conn->max_read_sge;
> > +
> > +	wr->send_wr_num = DIV_ROUND_UP(data->nents, max_sge);
> >   	wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num,
> >   				GFP_KERNEL);
> >   	if (!wr->send_wr) {
> > @@ -2463,11 +2495,11 @@ isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
> >   	}
> >
> >   	wr->isert_cmd = isert_cmd;
> > -	rdma_write_max = isert_conn->max_sge * PAGE_SIZE;
> >
> > +	rdma_max_len = max_sge * PAGE_SIZE;
> >   	for (i = 0; i < wr->send_wr_num; i++) {
> >   		send_wr = &isert_cmd->rdma_wr.send_wr[i];
> > -		data_len = min(data_left, rdma_write_max);
> > +		data_len = min(data_left, rdma_max_len);
> >
> >   		send_wr->send_flags = 0;
> >   		if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) {
> > @@ -2489,7 +2521,7 @@ isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
> >   		}
> >
> >   		ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,
> > -					send_wr, data_len, offset);
> > +					send_wr, data_len, offset, max_sge);
> >   		ib_sge += ib_sge_cnt;
> >
> >   		offset += data_len;
> > @@ -2618,6 +2650,13 @@ isert_fast_reg_mr(struct isert_conn *isert_conn,
> >   	fr_wr.wr.fast_reg.rkey = mr->rkey;
> >   	fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE;
> >
> > +	/*
> > +	 * IWARP transports need REMOTE_WRITE for MRs used as the target of
> > +	 * an RDMA_READ.
> > +	 */
> > +	if (rdma_protocol_iwarp(ib_dev, isert_conn->cm_id->port_num))
> > +		fr_wr.wr.fast_reg.access_flags |= IB_ACCESS_REMOTE_WRITE;
> > +
> >   	if (!wr)
> >   		wr = &fr_wr;
> >   	else
> > diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h
> > index 9ec23a7..47cf11b 100644
> > --- a/drivers/infiniband/ulp/isert/ib_isert.h
> > +++ b/drivers/infiniband/ulp/isert/ib_isert.h
> > @@ -153,6 +153,7 @@ struct isert_conn {
> >   	u32			initiator_depth;
> >   	bool			pi_support;
> >   	u32			max_sge;
> > +	u32			max_read_sge;
> >   	char			*login_buf;
> >   	char			*login_req_buf;
> >   	char			*login_rsp_buf;
> >
>