[PATCH 1/4] IB/core: add RAW Packet QP type

[PATCH 1/4] IB/core: add RAW Packet QP type

[Or Gerlitz]

Applications that use this qp type should build the whole packet,
including link level headers when sending. On the receive side, the
HW isn't expected to strip any headers. The new qp type designated
usage is from user-space in Ethernet environments, e.g by applications
that do TCP/IP offloading. Only processes with the NET_RAW capability
may open such qp. The name raw packet was selected to resemble the
similarity to AF_PACKET / SOL_RAW sockets.

Signed-off-by: Or Gerlitz <ogerlitz-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---
 drivers/infiniband/core/uverbs_cmd.c |    3 +++
 drivers/infiniband/core/verbs.c      |    1 +
 include/rdma/ib_verbs.h              |    2 +-
 3 files changed, 5 insertions(+), 1 deletions(-)

diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index b930da4..8722e96 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -1399,6 +1399,9 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;

+	if (cmd.qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
+		return -EPERM;
+
 	INIT_UDATA(&udata, buf + sizeof cmd,
 		   (unsigned long) cmd.response + sizeof resp,
 		   in_len - sizeof cmd, out_len - sizeof resp);
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 602b1bd..f73e15b 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -479,6 +479,7 @@ static const struct {
 				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
 						IB_QP_PORT			|
 						IB_QP_QKEY),
+				[IB_QPT_RAW_PACKET] = IB_QP_PORT,
 				[IB_QPT_UC]  = (IB_QP_PKEY_INDEX		|
 						IB_QP_PORT			|
 						IB_QP_ACCESS_FLAGS),
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index c3cca5a..6220b8b 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -605,7 +605,7 @@ enum ib_qp_type {
 	IB_QPT_UD,
 	IB_QPT_RAW_IPV6,
 	IB_QPT_RAW_ETHERTYPE,
-	/* Save 8 for RAW_PACKET */
+ 	IB_QPT_RAW_PACKET = 8,
 	IB_QPT_XRC_INI = 9,
 	IB_QPT_XRC_TGT,
 	IB_QPT_MAX
-- 
1.7.1


--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH 2/4] IB/mlx4: add Raw Packet QP support

[Or Gerlitz]

Implementation of the RAW Packet QP for the mlx4_ib driver,
use the MLX transport as done by the mlx4_en Ethernet driver.

Signed-off-by: Or Gerlitz <ogerlitz-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---
 drivers/infiniband/hw/mlx4/qp.c |   29 ++++++++++++++++++++++++++---
 1 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index aa2aefa..97ee02a 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -84,6 +84,11 @@ enum {
 	MLX4_IB_CACHE_LINE_SIZE	= 64,
 };

+enum {
+	MLX4_RAW_QP_MTU		= 7,
+	MLX4_RAW_QP_MSGMAX	= 31,
+};
+
 static const __be32 mlx4_ib_opcode[] = {
 	[IB_WR_SEND]				= cpu_to_be32(MLX4_OPCODE_SEND),
 	[IB_WR_LSO]				= cpu_to_be32(MLX4_OPCODE_LSO),
@@ -573,7 +578,12 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
 	if (sqpn) {
 		qpn = sqpn;
 	} else {
-		err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn);
+		/* Raw packet QPNs must be aligned to 8 bits. If not, the WQE
+		   BlueFlame setup flow would wrongly cause vlan insertion */
+		if (init_attr->qp_type == IB_QPT_RAW_PACKET)
+			err = mlx4_qp_reserve_range(dev->dev, 1, 1 << 8, &qpn);
+		else
+			err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn);
 		if (err)
 			goto err_wrid;
 	}
@@ -791,6 +801,7 @@ struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
 	case IB_QPT_RC:
 	case IB_QPT_UC:
 	case IB_QPT_UD:
+	case IB_QPT_RAW_PACKET:
 	{
 		qp = kzalloc(sizeof *qp, GFP_KERNEL);
 		if (!qp)
@@ -872,7 +883,8 @@ static int to_mlx4_st(enum ib_qp_type type)
 	case IB_QPT_XRC_INI:
 	case IB_QPT_XRC_TGT:	return MLX4_QP_ST_XRC;
 	case IB_QPT_SMI:
-	case IB_QPT_GSI:	return MLX4_QP_ST_MLX;
+	case IB_QPT_GSI:
+	case IB_QPT_RAW_PACKET:	return MLX4_QP_ST_MLX;
 	default:		return -1;
 	}
 }
@@ -1042,6 +1054,8 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,

 	if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
 		context->mtu_msgmax = (IB_MTU_4096 << 5) | 11;
+	else if (ibqp->qp_type == IB_QPT_RAW_PACKET)
+		context->mtu_msgmax = (MLX4_RAW_QP_MTU << 5) | MLX4_RAW_QP_MSGMAX;
 	else if (ibqp->qp_type == IB_QPT_UD) {
 		if (qp->flags & MLX4_IB_QP_LSO)
 			context->mtu_msgmax = (IB_MTU_4096 << 5) |
@@ -1200,7 +1214,8 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 	if (cur_state == IB_QPS_INIT &&
 	    new_state == IB_QPS_RTR  &&
 	    (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI ||
-	     ibqp->qp_type == IB_QPT_UD)) {
+	     ibqp->qp_type == IB_QPT_UD ||
+	     ibqp->qp_type == IB_QPT_RAW_PACKET)) {
 		context->pri_path.sched_queue = (qp->port - 1) << 6;
 		if (is_qp0(dev, qp))
 			context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE;
@@ -1319,6 +1334,14 @@ int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		goto out;
 	}

+	if ((attr_mask & IB_QP_PORT) && (ibqp->qp_type == IB_QPT_RAW_PACKET) &&
+		(rdma_port_get_link_layer(&dev->ib_dev, attr->port_num)
+				!= IB_LINK_LAYER_ETHERNET)) {
+		printk(KERN_WARNING "mlx4_ib: RAW PACKET QPs can "
+			"only be used over Ethernet ports\n");
+		goto out;
+	}
+
 	if (attr_mask & IB_QP_PKEY_INDEX) {
 		int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
 		if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p])
-- 
1.7.1


--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH libibverbs 3/4] add RAW Packet QP type

[Or Gerlitz]

Applications that use this qp type should build the whole packet,
including link level headers when sending. On the receive side, the
HW isn't expected to strip any headers. Only processes with the NET_RAW
capability may open such qp. The name raw packet was selected to
resemble the similarity to AF_PACKET / SOL_RAW sockets.

Signed-off-by: Or Gerlitz <ogerlitz-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---
 include/infiniband/verbs.h |    3 ++-
 man/ibv_create_qp.3        |    2 +-
 man/ibv_modify_qp.3        |   10 ++++++++++
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 6acfc81..8ed8a66 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -399,7 +399,8 @@ struct ibv_srq_init_attr {
 enum ibv_qp_type {
 	IBV_QPT_RC = 2,
 	IBV_QPT_UC,
-	IBV_QPT_UD
+	IBV_QPT_UD,
+	IBV_QPT_RAW_PACKET = 8
 };

 struct ibv_qp_cap {
diff --git a/man/ibv_create_qp.3 b/man/ibv_create_qp.3
index 5301ad8..7feeab2 100644
--- a/man/ibv_create_qp.3
+++ b/man/ibv_create_qp.3
@@ -28,7 +28,7 @@ struct ibv_cq          *send_cq;        /* CQ to be associated with the Send Que
 struct ibv_cq          *recv_cq;        /* CQ to be associated with the Receive Queue (RQ) */
 struct ibv_srq         *srq;            /* SRQ handle if QP is to be associated with an SRQ, otherwise NULL */
 struct ibv_qp_cap       cap;            /* QP capabilities */
-enum ibv_qp_type        qp_type;        /* QP Transport Service Type: IBV_QPT_RC, IBV_QPT_UC, or IBV_QPT_UD */
+enum ibv_qp_type        qp_type;        /* QP Transport Service Type: IBV_QPT_RC, IBV_QPT_UC, IBV_QPT_UD or IBV_QPT_RAW_PACKET */
 int                     sq_sig_all;     /* If set, each Work Request (WR) submitted to the SQ generates a completion entry */
 .in -8
 };
diff --git a/man/ibv_modify_qp.3 b/man/ibv_modify_qp.3
index 9eabcdf..cb3faaa 100644
--- a/man/ibv_modify_qp.3
+++ b/man/ibv_modify_qp.3
@@ -159,6 +159,16 @@ RTR  \fB          IBV_QP_STATE, IBV_QP_AV, IBV_QP_PATH_MTU, \fR
 RTS  \fB          IBV_QP_STATE, IBV_QP_SQ_PSN, IBV_QP_MAX_QP_RD_ATOMIC, \fR
      \fB          IBV_QP_RETRY_CNT, IBV_QP_RNR_RETRY, IBV_QP_TIMEOUT \fR
 .fi
+.PP
+.nf
+For QP Transport Service Type \fB IBV_QPT_RAW_PACKET\fR:
+.sp
+Next state     Required attributes
+\-\-\-\-\-\-\-\-\-\-     \-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-
+Init \fB          IBV_QP_STATE, IBV_QP_PORT\fR
+RTR  \fB          IBV_QP_STATE\fR
+RTS  \fB          IBV_QP_STATE\fR
+.fi
 .SH "SEE ALSO"
 .BR ibv_create_qp (3),
 .BR ibv_destroy_qp (3),
-- 
1.7.1



--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH libmlx4 4/4] add Raw Packet QP support

[Or Gerlitz]

Basic support for RAW Packet QP.

Signed-off-by: Or Gerlitz <ogerlitz-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---
 src/qp.c |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)

diff --git a/src/qp.c b/src/qp.c
index 40a6689..90c4e80 100644
--- a/src/qp.c
+++ b/src/qp.c
@@ -286,6 +286,10 @@ int mlx4_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
 			size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
 			break;

+		case IBV_QPT_RAW_PACKET:
+			/* For raw eth, the MLX4_WQE_CTRL_SOLICIT flag is used
+			 * to indicate that no icrc should be calculated */
+			ctrl->srcrb_flags |= htonl(MLX4_WQE_CTRL_SOLICIT);
 		default:
 			break;
 		}
-- 
1.6.5.5


--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 0/4] add RAW Packet QP type

[Steve Wise]

On 01/17/2012 05:34 AM, Or Gerlitz wrote:
> The new qp type designated usage is from user-space in Ethernet environments,
> e.g by applications that do TCP/IP offloading. Only processes with the NET_RAW
> capability may open such qp. The name raw packet was selected to resemble the
> similarity to AF_PACKET / SOL_RAW sockets. Applications that use this qp type
> should deal with whole packets, including link level headers.
>
> This series allows to create such QPs and send packets over them. To receive
> packets, flow steering support has to be added to the verbs and low-level
> drivers. Flow Steering is the ability to program the HCA to direct a packet
> which matches a given flow specification to a given QP. Flow specs set by
> applications are typically made of L3 (IP) and L4 (TCP/UDP) based tuples,
> where network drivers typically use L2 based tuples. Core and mlx4 patches
> for flow steering are expected in the coming weeks.

Hey Or,

I think this series should add some new send flags for HW that does checksum offload:

For example, cxgb4 supports these:

enum {                     /* TX_PKT_XT checksum types */
         TX_CSUM_TCP    = 0,
         TX_CSUM_UDP    = 1,
         TX_CSUM_CRC16  = 4,
         TX_CSUM_CRC32  = 5,
         TX_CSUM_CRC32C = 6,
         TX_CSUM_FCOE   = 7,
         TX_CSUM_TCPIP  = 8,
         TX_CSUM_UDPIP  = 9,
         TX_CSUM_TCPIP6 = 10,
         TX_CSUM_UDPIP6 = 11,
         TX_CSUM_IP     = 12,
};

I'm sure mlx4 has this sort of functionality too?

Another form of HW assist is with VLAN insertion/extraction.  The API should provide a way to specify if a VLAN ID 
should be inserted by HW and removed from a packet on ingress (and passed to the app via the CQE).  In fact, we probably 
want a way to associate a VLAN with a RAW QP, maybe as a QP attribute?

Also, on ingress, most hardware can do INET checksum validation, and a way to indicate the results to the application is 
needed.  Perhaps flags in the CQE?  The cxgb4 device provides many fields on a ingress packet completion that would be 
useful for user mode applications including indications of MAC RX errors, protocol length vs packet length mismatches, 
IP version not 4 or 6, and more.  Does mlx4 has these sorts of indications on ingress packet CQEs?

Food for thought.

Steve.



--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [PATCH 0/4] add RAW Packet QP type

[Walukiewicz, Miroslaw]

According VLANs, for RAW QP better solution is allowing inserting VLANs per packet by adding new flags to ib_post_send with special field containing VLAN ID.

On ingress, it would be interesting to see the ingress VLAN in CQE, by introducing a new field and flags indicating VLAN appearance.

As Steve mentioned, the HW checksum offload is necessary also in the API for sending IP fragments. When IP packet is not fragmented it is necessary to send it with L4/L3 sum computed by HW. And when fragments are sent the L3 only csum computation is requested.

Regards,

Mirek


-----Original Message-----
From: linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org [mailto:linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org] On Behalf Of Steve Wise
Sent: Tuesday, January 17, 2012 4:08 PM
To: Or Gerlitz
Cc: Roland Dreier; linux-rdma; Christoph Lameter; Liran Liss
Subject: Re: [PATCH 0/4] add RAW Packet QP type


On 01/17/2012 05:34 AM, Or Gerlitz wrote:
> The new qp type designated usage is from user-space in Ethernet environments,
> e.g by applications that do TCP/IP offloading. Only processes with the NET_RAW
> capability may open such qp. The name raw packet was selected to resemble the
> similarity to AF_PACKET / SOL_RAW sockets. Applications that use this qp type
> should deal with whole packets, including link level headers.
>
> This series allows to create such QPs and send packets over them. To receive
> packets, flow steering support has to be added to the verbs and low-level
> drivers. Flow Steering is the ability to program the HCA to direct a packet
> which matches a given flow specification to a given QP. Flow specs set by
> applications are typically made of L3 (IP) and L4 (TCP/UDP) based tuples,
> where network drivers typically use L2 based tuples. Core and mlx4 patches
> for flow steering are expected in the coming weeks.

Hey Or,

I think this series should add some new send flags for HW that does checksum offload:

For example, cxgb4 supports these:

enum {                     /* TX_PKT_XT checksum types */
         TX_CSUM_TCP    = 0,
         TX_CSUM_UDP    = 1,
         TX_CSUM_CRC16  = 4,
         TX_CSUM_CRC32  = 5,
         TX_CSUM_CRC32C = 6,
         TX_CSUM_FCOE   = 7,
         TX_CSUM_TCPIP  = 8,
         TX_CSUM_UDPIP  = 9,
         TX_CSUM_TCPIP6 = 10,
         TX_CSUM_UDPIP6 = 11,
         TX_CSUM_IP     = 12,
};

I'm sure mlx4 has this sort of functionality too?

Another form of HW assist is with VLAN insertion/extraction.  The API should provide a way to specify if a VLAN ID 
should be inserted by HW and removed from a packet on ingress (and passed to the app via the CQE).  In fact, we probably 
want a way to associate a VLAN with a RAW QP, maybe as a QP attribute?

Also, on ingress, most hardware can do INET checksum validation, and a way to indicate the results to the application is 
needed.  Perhaps flags in the CQE?  The cxgb4 device provides many fields on a ingress packet completion that would be 
useful for user mode applications including indications of MAC RX errors, protocol length vs packet length mismatches, 
IP version not 4 or 6, and more.  Does mlx4 has these sorts of indications on ingress packet CQEs?

Food for thought.

Steve.



--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 0/4] add RAW Packet QP type

[Or Gerlitz]

On 1/17/2012 5:08 PM, Steve Wise wrote:
> I think this series should add some new send flags for HW that does 
> checksum offload [...] also, on ingress, most hardware can do INET 
> checksum validation, and a way to indicate the results to the 
> application is needed.  Perhaps flags in the CQE? [...] another form 
> of HW assist is with VLAN insertion/extraction.  The API should 
> provide a way to specify if a VLAN ID should be inserted by HW and 
> removed from a packet on ingress (and passed to the app via the CQE).  
> In fact, we probably want a way to associate a VLAN with a RAW QP, 
> maybe as a QP attribute?

Hi Steve,

Nice to see how a discussion is quickly revived when the subject is 
close to people's needs... anyway, yep, sure, this submission allows for 
basic functionality, as I mentioned and there's more to add here, but 
this could (should...) be done gradually,  i.e in steps that adds on the 
basic patch. Now to the points you've raised:

Sure, we want to be able to do checksum offload on TX and on RX as well. 
Adding checksum offload support will be done with a patch to libibverbs 
which is similar in spirit to commit e0605d "IB/core: Add IP checksum 
offload support" subject to reporting the RX checksum okay through new 
IB_WC_IP_CSUM_OK bit in ibv_wc->wc_flags along the lines of the patch I 
sent last week to the kernel. So we will have a new device capability 
and two new bit flags IB_SEND_IP_CSUM and IB_WC_IP_CSUM_OK, no ABI 
breaking... happy.

Less simple will be to add support in libibverbs and the low level 
driver libraries, for Large-Send-Offload, e.g in the spirit of commits 
b846f25aa "IB/core: Add creation flags to struct ib_qp_init_attr" and 
c93570f2 "IB/core: Add IPoIB UD LSO support", since we probably need to 
be able to specify something in the qp creation (Sean's verbs extension 
proposal!) and add fields to struct ibv_send_wr.wr.ud, in the WR case, 
we need more 16 (= 8 + 4 + 4) bytes, where it seems before the patch we 
have 12 bytes extra from wr.atomic, I need to check the compiler packing 
here...

> diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
> index 6acfc81..81bc55f 100644
> --- a/include/infiniband/verbs.h
> +++ b/include/infiniband/verbs.h
> @@ -534,6 +534,9 @@ struct ibv_send_wr {
>                         struct ibv_ah  *ah;
>                         uint32_t        remote_qpn;
>                         uint32_t        remote_qkey;
> +                       void            *header;
> +                       uint32_t        hlen;
> +                       uint32_t        mss;
>                 } ud;
>         } wr;
>  };


As for vlan stripping/insertion, I'm not having definitive opinion yet - 
if we don't expect the system (library, firmware, hardware) to 
insert/strip the L2 MAC header, I', not sure what does it buy us to have 
the vlan tag inserted/stripped?

As for your comment on vlan association with raw qp, do you actually 
refer to the steering work the HCA/NIC should do w.r.t that qp? e.g 
you'd like to have L2 elements in the tuple/rule which is used to steer 
packets to that QP?

Or.


--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 0/4] add RAW Packet QP type

[Steve Wise]

On 01/17/2012 09:59 AM, Or Gerlitz wrote:
> On 1/17/2012 5:08 PM, Steve Wise wrote:
>> I think this series should add some new send flags for HW that does checksum offload [...] also, on ingress, most 
>> hardware can do INET checksum validation, and a way to indicate the results to the application is needed.  Perhaps 
>> flags in the CQE? [...] another form of HW assist is with VLAN insertion/extraction.  The API should provide a way to 
>> specify if a VLAN ID should be inserted by HW and removed from a packet on ingress (and passed to the app via the 
>> CQE).  In fact, we probably want a way to associate a VLAN with a RAW QP, maybe as a QP attribute?
>
> Hi Steve,
>
> Nice to see how a discussion is quickly revived when the subject is close to people's needs... anyway, yep, sure, this 
> submission allows for basic functionality, as I mentioned and there's more to add here, but this could (should...) be 
> done gradually,  i.e in steps that adds on the basic patch. Now to the points you've raised:
>
> Sure, we want to be able to do checksum offload on TX and on RX as well. Adding checksum offload support will be done 
> with a patch to libibverbs which is similar in spirit to commit e0605d "IB/core: Add IP checksum offload support" 
> subject to reporting the RX checksum okay through new IB_WC_IP_CSUM_OK bit in ibv_wc->wc_flags along the lines of the 
> patch I sent last week to the kernel. So we will have a new device capability and two new bit flags IB_SEND_IP_CSUM 
> and IB_WC_IP_CSUM_OK, no ABI breaking... happy.

Sounds good, with the caveat that we need more bits to specify L4 CSUM generation as well as L3 (and L2 CRC).

>
>
> Less simple will be to add support in libibverbs and the low level driver libraries, for Large-Send-Offload, e.g in 
> the spirit of commits b846f25aa "IB/core: Add creation flags to struct ib_qp_init_attr" and c93570f2 "IB/core: Add 
> IPoIB UD LSO support", since we probably need to be able to specify something in the qp creation (Sean's verbs 
> extension proposal!) and add fields to struct ibv_send_wr.wr.ud, in the WR case, we need more 16 (= 8 + 4 + 4) bytes, 
> where it seems before the patch we have 12 bytes extra from wr.atomic, I need to check the compiler packing here...
>
>> diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
>> index 6acfc81..81bc55f 100644
>> --- a/include/infiniband/verbs.h
>> +++ b/include/infiniband/verbs.h
>> @@ -534,6 +534,9 @@ struct ibv_send_wr {
>>                         struct ibv_ah  *ah;
>>                         uint32_t        remote_qpn;
>>                         uint32_t        remote_qkey;
>> +                       void            *header;
>> +                       uint32_t        hlen;
>> +                       uint32_t        mss;
>>                 } ud;
>>         } wr;
>>  };
>
>
> As for vlan stripping/insertion, I'm not having definitive opinion yet - if we don't expect the system (library, 
> firmware, hardware) to insert/strip the L2 MAC header, I', not sure what does it buy us to have the vlan tag 
> inserted/stripped?

Actually, I'm not sure.  But cxgb4 allows for it. :)  Maybe someone from Chelsio could explain the utility?

>
> As for your comment on vlan association with raw qp, do you actually refer to the steering work the HCA/NIC should do 
> w.r.t that qp? e.g you'd like to have L2 elements in the tuple/rule which is used to steer packets to that QP?
>

It definitely needs to be part of the steering filter/flow.  If we define a new API for adding the filter/flow, then I 
think that is one place where the VLAN ID would be provided by the app.  Also as part of a SEND WR, as Miroslaw points 
out.  My original thought was that you associate it with the RAW QP and then it is used by the provider as needed, but 
maybe that is too restrictive.


Steve.

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [PATCH 2/4] IB/mlx4: add Raw Packet QP support

[Hefty, Sean]

> +enum {
> +	MLX4_RAW_QP_MTU		= 7,

Is there any special meaning for MTU 7 that should be standardized in ib_verbs.h?

In other places, the code uses IB_MTU_4096 (5):

> @@ -1042,6 +1054,8 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
> 
>  	if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
>  		context->mtu_msgmax = (IB_MTU_4096 << 5) | 11;
> +	else if (ibqp->qp_type == IB_QPT_RAW_PACKET)
> +		context->mtu_msgmax = (MLX4_RAW_QP_MTU << 5) | MLX4_RAW_QP_MSGMAX;
>  	else if (ibqp->qp_type == IB_QPT_UD) {
>  		if (qp->flags & MLX4_IB_QP_LSO)
>  			context->mtu_msgmax = (IB_MTU_4096 << 5) |
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 2/4] IB/mlx4: add Raw Packet QP support

[Or Gerlitz]

On Wed, Jan 18, 2012 at 8:47 PM, Hefty, Sean <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org> wrote:
>> +     MLX4_RAW_QP_MTU         = 7,

> Is there any special meaning for MTU 7 that should be standardized in ib_verbs.h?
> In other places, the code uses IB_MTU_4096 (5):

nope, its the ConnectX firmware that dictates this value for RAW QPs,
same for the msgmax
being 31.

Or.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [PATCH 1/4] IB/core: add RAW Packet QP type

[Hefty, Sean]

> Applications that use this qp type should build the whole packet,
> including link level headers when sending. On the receive side, the
> HW isn't expected to strip any headers. The new qp type designated
> usage is from user-space in Ethernet environments, e.g by applications
> that do TCP/IP offloading. Only processes with the NET_RAW capability
> may open such qp. The name raw packet was selected to resemble the
> similarity to AF_PACKET / SOL_RAW sockets.
> 
> Signed-off-by: Or Gerlitz <ogerlitz-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>

Reviewed-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>

> ---
>  drivers/infiniband/core/uverbs_cmd.c |    3 +++
>  drivers/infiniband/core/verbs.c      |    1 +
>  include/rdma/ib_verbs.h              |    2 +-
>  3 files changed, 5 insertions(+), 1 deletions(-)
> 
> diff --git a/drivers/infiniband/core/uverbs_cmd.c
> b/drivers/infiniband/core/uverbs_cmd.c
> index b930da4..8722e96 100644
> --- a/drivers/infiniband/core/uverbs_cmd.c
> +++ b/drivers/infiniband/core/uverbs_cmd.c
> @@ -1399,6 +1399,9 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
>  	if (copy_from_user(&cmd, buf, sizeof cmd))
>  		return -EFAULT;
> 
> +	if (cmd.qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
> +		return -EPERM;
> +
>  	INIT_UDATA(&udata, buf + sizeof cmd,
>  		   (unsigned long) cmd.response + sizeof resp,
>  		   in_len - sizeof cmd, out_len - sizeof resp);
> diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
> index 602b1bd..f73e15b 100644
> --- a/drivers/infiniband/core/verbs.c
> +++ b/drivers/infiniband/core/verbs.c
> @@ -479,6 +479,7 @@ static const struct {
>  				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
>  						IB_QP_PORT			|
>  						IB_QP_QKEY),
> +				[IB_QPT_RAW_PACKET] = IB_QP_PORT,
>  				[IB_QPT_UC]  = (IB_QP_PKEY_INDEX		|
>  						IB_QP_PORT			|
>  						IB_QP_ACCESS_FLAGS),
> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
> index c3cca5a..6220b8b 100644
> --- a/include/rdma/ib_verbs.h
> +++ b/include/rdma/ib_verbs.h
> @@ -605,7 +605,7 @@ enum ib_qp_type {
>  	IB_QPT_UD,
>  	IB_QPT_RAW_IPV6,
>  	IB_QPT_RAW_ETHERTYPE,
> -	/* Save 8 for RAW_PACKET */
> + 	IB_QPT_RAW_PACKET = 8,
>  	IB_QPT_XRC_INI = 9,
>  	IB_QPT_XRC_TGT,
>  	IB_QPT_MAX
> --
> 1.7.1
> 
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 1/4] IB/core: add RAW Packet QP type

[Or Gerlitz]

On Thu, Jan 19, 2012 at 3:28 AM, Hefty, Sean <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org> wrote:
>> Applications that use this qp type should build the whole packet,
>> including link level headers when sending. On the receive side, the
>> HW isn't expected to strip any headers. The new qp type designated
>> usage is from user-space in Ethernet environments, e.g by applications
>> that do TCP/IP offloading. Only processes with the NET_RAW capability
>> may open such qp. The name raw packet was selected to resemble the
>> similarity to AF_PACKET / SOL_RAW sockets.

>> Signed-off-by: Or Gerlitz <ogerlitz-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>

> Reviewed-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>

Roland,

Also here, Sean provided his reviewed-by signature, people (CCed
Christoph and others) keep asking me about this patch set and I didn't
get any feedback from you.

Or.


>>  drivers/infiniband/core/uverbs_cmd.c |    3 +++
>>  drivers/infiniband/core/verbs.c      |    1 +
>>  include/rdma/ib_verbs.h              |    2 +-
>>  3 files changed, 5 insertions(+), 1 deletions(-)
>>
>> diff --git a/drivers/infiniband/core/uverbs_cmd.c
>> b/drivers/infiniband/core/uverbs_cmd.c
>> index b930da4..8722e96 100644
>> --- a/drivers/infiniband/core/uverbs_cmd.c
>> +++ b/drivers/infiniband/core/uverbs_cmd.c
>> @@ -1399,6 +1399,9 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
>>       if (copy_from_user(&cmd, buf, sizeof cmd))
>>               return -EFAULT;
>>
>> +     if (cmd.qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
>> +             return -EPERM;
>> +
>>       INIT_UDATA(&udata, buf + sizeof cmd,
>>                  (unsigned long) cmd.response + sizeof resp,
>>                  in_len - sizeof cmd, out_len - sizeof resp);
>> diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
>> index 602b1bd..f73e15b 100644
>> --- a/drivers/infiniband/core/verbs.c
>> +++ b/drivers/infiniband/core/verbs.c
>> @@ -479,6 +479,7 @@ static const struct {
>>                               [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
>>                                               IB_QP_PORT                      |
>>                                               IB_QP_QKEY),
>> +                             [IB_QPT_RAW_PACKET] = IB_QP_PORT,
>>                               [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
>>                                               IB_QP_PORT                      |
>>                                               IB_QP_ACCESS_FLAGS),
>> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
>> index c3cca5a..6220b8b 100644
>> --- a/include/rdma/ib_verbs.h
>> +++ b/include/rdma/ib_verbs.h
>> @@ -605,7 +605,7 @@ enum ib_qp_type {
>>       IB_QPT_UD,
>>       IB_QPT_RAW_IPV6,
>>       IB_QPT_RAW_ETHERTYPE,
>> -     /* Save 8 for RAW_PACKET */
>> +     IB_QPT_RAW_PACKET = 8,
>>       IB_QPT_XRC_INI = 9,
>>       IB_QPT_XRC_TGT,
>>       IB_QPT_MAX
>> --
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 1/4] IB/core: add RAW Packet QP type

[Christoph Lameter]

On Thu, 16 Feb 2012, Or Gerlitz wrote:

> Also here, Sean provided his reviewed-by signature, people (CCed
> Christoph and others) keep asking me about this patch set and I didn't
> get any feedback from you.

I sure would like to see this merged for 3.4


--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH V1 1/4] IB/core: add RAW Packet QP type

[Or Gerlitz]

Applications that use this qp type should build the whole packet,
including L2 headers when sending. On the receive side, the HW isn't
expected to strip any headers. The new qp type designated usage is
from user-space in Ethernet environments, e.g by applications that do
TCP/IP offloading. Only processes with the NET_RAW capability may open
such qp. The name raw packet was selected to resemble the similarity
to AF_PACKET / SOL_RAW sockets.

Signed-off-by: Or Gerlitz <ogerlitz-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---

changes from V0:
 removed spaces at ib_verbs.h that went in by mistake

 drivers/infiniband/core/uverbs_cmd.c |    3 +++
 drivers/infiniband/core/verbs.c      |    1 +
 include/rdma/ib_verbs.h              |    2 +-
 3 files changed, 5 insertions(+), 1 deletions(-)

diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 4d27e4c..7d801e6 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -1399,6 +1399,9 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;

+	if (cmd.qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
+		return -EPERM;
+
 	INIT_UDATA(&udata, buf + sizeof cmd,
 		   (unsigned long) cmd.response + sizeof resp,
 		   in_len - sizeof cmd, out_len - sizeof resp);
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 575b780..140f9a5 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -479,6 +479,7 @@ static const struct {
 				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
 						IB_QP_PORT			|
 						IB_QP_QKEY),
+				[IB_QPT_RAW_PACKET] = IB_QP_PORT,
 				[IB_QPT_UC]  = (IB_QP_PKEY_INDEX		|
 						IB_QP_PORT			|
 						IB_QP_ACCESS_FLAGS),
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index c3cca5a..a3fa323 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -605,7 +605,7 @@ enum ib_qp_type {
 	IB_QPT_UD,
 	IB_QPT_RAW_IPV6,
 	IB_QPT_RAW_ETHERTYPE,
-	/* Save 8 for RAW_PACKET */
+	IB_QPT_RAW_PACKET = 8,
 	IB_QPT_XRC_INI = 9,
 	IB_QPT_XRC_TGT,
 	IB_QPT_MAX
-- 
1.7.1


--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 1/4] IB/core: add RAW Packet QP type

[Or Gerlitz]

On 2/16/2012 5:27 PM, Christoph Lameter wrote:
> On Thu, 16 Feb 2012, Or Gerlitz wrote:
>
>> Also here, Sean provided his reviewed-by signature, people (CCed Christoph and others) keep asking me about this patch set and I didn't get any feedback from you.
>
> I sure would like to see this merged for 3.4
>
>

Hi Roland,

Same here, I submitted the patches ~two months ago and haven't heard 
from you, I would be happy to see this going into 3.4 and if 
fixes/changes are needed please let me know as soon as possible so I 
will do them.

Or.

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 1/4] IB/core: add RAW Packet QP type

[Christoph Lameter]

On Mon, 12 Mar 2012, Or Gerlitz wrote:

> Same here, I submitted the patches ~two months ago and haven't heard from you,
> I would be happy to see this going into 3.4 and if fixes/changes are needed
> please let me know as soon as possible so I will do them.

I can confirm that the patches in OFED to the same effect work very well.
Widely deployed here.

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 1/4] IB/core: add RAW Packet QP type

[Roland Dreier]

Thanks, applied at long last.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 0/4] add RAW Packet QP type

[Shawn Bohrer]

On Tue, Jan 17, 2012 at 10:21:28AM -0600, Steve Wise wrote:
> On 01/17/2012 09:59 AM, Or Gerlitz wrote:
> >On 1/17/2012 5:08 PM, Steve Wise wrote:
> >>I think this series should add some new send flags for HW that
> >>does checksum offload [...] also, on ingress, most hardware can
> >>do INET checksum validation, and a way to indicate the results
> >>to the application is needed.  Perhaps flags in the CQE? [...]
> >>another form of HW assist is with VLAN insertion/extraction.
> >>The API should provide a way to specify if a VLAN ID should be
> >>inserted by HW and removed from a packet on ingress (and passed
> >>to the app via the CQE).  In fact, we probably want a way to
> >>associate a VLAN with a RAW QP, maybe as a QP attribute?
> >
> >Hi Steve,
> >
> >Nice to see how a discussion is quickly revived when the subject
> >is close to people's needs... anyway, yep, sure, this submission
> >allows for basic functionality, as I mentioned and there's more to
> >add here, but this could (should...) be done gradually,  i.e in
> >steps that adds on the basic patch. Now to the points you've
> >raised:
> >
> >Sure, we want to be able to do checksum offload on TX and on RX as
> >well. Adding checksum offload support will be done with a patch to
> >libibverbs which is similar in spirit to commit e0605d "IB/core:
> >Add IP checksum offload support" subject to reporting the RX
> >checksum okay through new IB_WC_IP_CSUM_OK bit in ibv_wc->wc_flags
> >along the lines of the patch I sent last week to the kernel. So we
> >will have a new device capability and two new bit flags
> >IB_SEND_IP_CSUM and IB_WC_IP_CSUM_OK, no ABI breaking... happy.
> 
> Sounds good, with the caveat that we need more bits to specify L4 CSUM generation as well as L3 (and L2 CRC).

Hi Or,

I appologize if I missed it, but did any support for L3/L4 CSUM
generation get added?  Doesn't look like the upstream libibverbs has
it, and I don't seem to see any patches floating around.

Thanks,
Shawn
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 0/4] add RAW Packet QP type

[Or Gerlitz]

On Tue, May 21, 2013 at 1:43 AM, Shawn Bohrer <shawn.bohrer-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org> wrote:
> I appologize if I missed it, but did any support for L3/L4 CSUM
> generation get added?  Doesn't look like the upstream libibverbs has
> it, and I don't seem to see any patches floating around.

Roland commented that he will make a point release  this week for
libibverbs and libmlx4, the patches for CSUM offload I will post after
that release.

Or.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html