[PATCH 0/2] ConnectX-4/Connect-IB Port Counters

[PATCH 0/2] ConnectX-4/Connect-IB Port Counters

[Leon Romanovsky]

This is supplementary part of mlx5 driver to generic IB counters infrastructure.

Mark Bloch (2):
  IB/mlx5: Add per port counters
  IB/mlx5: Add port protocol stats

 drivers/infiniband/hw/mlx5/main.c    | 134 ++++++++++++++++++++++++++++++++++-
 drivers/infiniband/hw/mlx5/mlx5_ib.h |   6 ++
 drivers/infiniband/hw/mlx5/qp.c      |   9 +++
 3 files changed, 147 insertions(+), 2 deletions(-)

-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH 1/2] IB/mlx5: Add per port counters

[Leon Romanovsky]

From: Mark Bloch <markb-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>

In order to support statistics for ports, we attach
each QP to a counter set which is dedicate to this port.

Signed-off-by: Mark Bloch <markb-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
Reviewed-by: Leon Romanovsky <leonro-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
Signed-off-by: Leon Romanovsky <leon-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org>
---
 drivers/infiniband/hw/mlx5/main.c    | 56 ++++++++++++++++++++++++++++++++++--
 drivers/infiniband/hw/mlx5/mlx5_ib.h |  6 ++++
 drivers/infiniband/hw/mlx5/qp.c      |  9 ++++++
 3 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index c72797c..e525891 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -2293,6 +2293,41 @@ static void mlx5_disable_roce(struct mlx5_ib_dev *dev)
 	unregister_netdevice_notifier(&dev->roce.nb);
 }
 
+static void mlx5_ib_dealloc_q_counters(struct mlx5_ib_dev *dev)
+{
+	unsigned int i;
+
+	for (i = 0; i < dev->num_ports; i++)
+		mlx5_core_dealloc_q_counter(dev->mdev,
+					    dev->port[i].q_cnt_id);
+}
+
+static int mlx5_ib_alloc_q_counters(struct mlx5_ib_dev *dev)
+{
+	int i;
+	int ret;
+
+	for (i = 0; i < dev->num_ports; i++) {
+		ret = mlx5_core_alloc_q_counter(dev->mdev,
+						&dev->port[i].q_cnt_id);
+		if (ret) {
+			mlx5_ib_warn(dev,
+				     "couldn't allocate queue counter for port %d, err %d\n",
+				     i + 1, ret);
+			goto dealloc_counters;
+		}
+	}
+
+	return 0;
+
+dealloc_counters:
+	while (--i >= 0)
+		mlx5_core_dealloc_q_counter(dev->mdev,
+					    dev->port[i].q_cnt_id);
+
+	return ret;
+}
+
 static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 {
 	struct mlx5_ib_dev *dev;
@@ -2315,10 +2350,15 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 
 	dev->mdev = mdev;
 
+	dev->port = kcalloc(MLX5_CAP_GEN(mdev, num_ports), sizeof(*dev->port),
+			    GFP_KERNEL);
+	if (!dev->port)
+		goto err_dealloc;
+
 	rwlock_init(&dev->roce.netdev_lock);
 	err = get_port_caps(dev);
 	if (err)
-		goto err_dealloc;
+		goto err_free_port;
 
 	if (mlx5_use_mad_ifc(dev))
 		get_ext_port_caps(dev);
@@ -2467,10 +2507,14 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 	if (err)
 		goto err_rsrc;
 
-	err = ib_register_device(&dev->ib_dev, NULL);
+	err = mlx5_ib_alloc_q_counters(dev);
 	if (err)
 		goto err_odp;
 
+	err = ib_register_device(&dev->ib_dev, NULL);
+	if (err)
+		goto err_q_cnt;
+
 	err = create_umr_res(dev);
 	if (err)
 		goto err_dev;
@@ -2492,6 +2536,9 @@ err_umrc:
 err_dev:
 	ib_unregister_device(&dev->ib_dev);
 
+err_q_cnt:
+	mlx5_ib_dealloc_q_counters(dev);
+
 err_odp:
 	mlx5_ib_odp_remove_one(dev);
 
@@ -2502,6 +2549,9 @@ err_disable_roce:
 	if (ll == IB_LINK_LAYER_ETHERNET)
 		mlx5_disable_roce(dev);
 
+err_free_port:
+	kfree(dev->port);
+
 err_dealloc:
 	ib_dealloc_device((struct ib_device *)dev);
 
@@ -2514,11 +2564,13 @@ static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev, 1);
 
 	ib_unregister_device(&dev->ib_dev);
+	mlx5_ib_dealloc_q_counters(dev);
 	destroy_umrc_res(dev);
 	mlx5_ib_odp_remove_one(dev);
 	destroy_dev_resources(&dev->devr);
 	if (ll == IB_LINK_LAYER_ETHERNET)
 		mlx5_disable_roce(dev);
+	kfree(dev->port);
 	ib_dealloc_device(&dev->ib_dev);
 }
 
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index c4a9825..930a9a8 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -546,6 +546,10 @@ struct mlx5_ib_resources {
 	struct mutex	mutex;
 };
 
+struct mlx5_ib_port {
+	u16 q_cnt_id;
+};
+
 struct mlx5_roce {
 	/* Protect mlx5_ib_get_netdev from invoking dev_hold() with a NULL
 	 * netdev pointer
@@ -581,6 +585,8 @@ struct mlx5_ib_dev {
 	struct srcu_struct      mr_srcu;
 #endif
 	struct mlx5_ib_flow_db	flow_db;
+	/* Array with num_ports elements */
+	struct mlx5_ib_port	*port;
 };
 
 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 5041176..c770d00 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -2343,6 +2343,15 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	else
 		sqd_event = 0;
 
+	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
+		u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num :
+			       qp->port) - 1;
+		struct mlx5_ib_port *mibport = &dev->port[port_num];
+
+		context->qp_counter_set_usr_page |=
+			cpu_to_be32(mibport->q_cnt_id << 16);
+	}
+
 	if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
 		context->sq_crq_size |= cpu_to_be16(1 << 4);
 
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH 2/2] IB/mlx5: Add port protocol stats

[Leon Romanovsky]

From: Mark Bloch <markb-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>

Expose new counters using the get protocol stats callback.
We expose the following counters:

|------------------------------------------------------------------------|
|      Name           | IB | EN |           Description                  |
|------------------------------------------------------------------------|
|rx_write_requests    | +  | -  | Number of received WRITE requests for  |
|                     |    |    | the associated QP.                     |
|------------------------------------------------------------------------|
|rx_read_requests     | +  | -  | Number of received READ requests for   |
|                     |    |    | the associated QP.                     |
|------------------------------------------------------------------------|
|rx_atomic_requests   | +  | -  | Number of received ATOMIC requests for |
|                     |    |    | the associated QP.                     |
|------------------------------------------------------------------------|
|out_of_buffer        | +  | +  | Number of drops occurred due to lack   |
|                     |    |    | of WQE for the associated QPs/RQs.     |
|------------------------------------------------------------------------|
|out_of_sequence      | +  | -  | Number of errors in the packet         |
|                     |    |    | transport sequence number              |
|------------------------------------------------------------------------|
|duplicate_request    | +  | +  | Number of received duplicated packets. |
|                     |    |    | A request that previously executed is  |
|                     |    |    | named duplicated.                      |
|------------------------------------------------------------------------|
|rnr_nak_retry_err    | +  | +  | Number of received RNR NAC packets.    |
|                     |    |    | The QP retry limit did not exceed.     |
|------------------------------------------------------------------------|
|packet_seq_err       | +  | +  | Number of received NAK - sequence error|
|                     |    |    | packets. The QP retry limit did not    |
|                     |    |    | exceed.                                |
|------------------------------------------------------------------------|
|implied_nak_err      | +  | +  | Number of times the requester detected |
|                     |    |    | an ACK with a PSN larger than expected |
|                     |    |    | PSN for RDMA READ or ATOMIC response   |
|                     |    |    | The QP retry limit did not exceed.     |
|------------------------------------------------------------------------|
|local_ack_timeout_err| +  | -  | Number of NO ACK responses from        |
|                     |    |    | responder within timer interval.       |
|                     |    |    | The QP retry limit did not exceed.     |
|------------------------------------------------------------------------|

Counters are available if all of them are supported.

Signed-off-by: Mark Bloch <markb-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
Signed-off-by: Leon Romanovsky <leonro-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
Signed-off-by: Leon Romanovsky <leon-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org>
---
 drivers/infiniband/hw/mlx5/main.c | 78 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index e525891..c392f08 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -2328,6 +2328,78 @@ dealloc_counters:
 	return ret;
 }
 
+static const char const *names[] = {
+	"rx_write_requests",
+	"rx_read_requests",
+	"rx_atomic_requests",
+	"out_of_buffer",
+	"out_of_sequence",
+	"duplicate_request",
+	"rnr_nak_retry_err",
+	"packet_seq_err",
+	"implied_nak_seq_err",
+	"local_ack_timeout_err",
+};
+
+static const size_t stats_offsets[] = {
+	MLX5_BYTE_OFF(query_q_counter_out, rx_write_requests),
+	MLX5_BYTE_OFF(query_q_counter_out, rx_read_requests),
+	MLX5_BYTE_OFF(query_q_counter_out, rx_atomic_requests),
+	MLX5_BYTE_OFF(query_q_counter_out, out_of_buffer),
+	MLX5_BYTE_OFF(query_q_counter_out, out_of_sequence),
+	MLX5_BYTE_OFF(query_q_counter_out, duplicate_request),
+	MLX5_BYTE_OFF(query_q_counter_out, rnr_nak_retry_err),
+	MLX5_BYTE_OFF(query_q_counter_out, packet_seq_err),
+	MLX5_BYTE_OFF(query_q_counter_out, implied_nak_seq_err),
+	MLX5_BYTE_OFF(query_q_counter_out, local_ack_timeout_err),
+};
+
+static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev,
+						    u8 port_num)
+{
+	BUILD_BUG_ON(ARRAY_SIZE(names) != ARRAY_SIZE(stats_offsets));
+
+	/* We support only per port stats */
+	if (port_num == 0)
+		return NULL;
+
+	return rdma_alloc_hw_stats_struct(names, ARRAY_SIZE(names),
+					  RDMA_HW_STATS_DEFAULT_LIFESPAN);
+}
+
+static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
+				struct rdma_hw_stats *stats,
+				u8 port, int index)
+{
+	struct mlx5_ib_dev *dev = to_mdev(ibdev);
+	int outlen = MLX5_ST_SZ_BYTES(query_q_counter_out);
+	void *out;
+	__be32 val;
+	int ret;
+	int i;
+
+	if (!port || !stats)
+		return -ENOSYS;
+
+	out = mlx5_vzalloc(outlen);
+	if (!out)
+		return -ENOMEM;
+
+	ret = mlx5_core_query_q_counter(dev->mdev,
+					dev->port[port - 1].q_cnt_id, 0,
+					out, outlen);
+	if (ret)
+		goto free;
+
+	for (i = 0; i < ARRAY_SIZE(names); i++) {
+		val = *(__be32 *)(out + stats_offsets[i]);
+		stats->value[i] = (u64)be32_to_cpu(val);
+	}
+free:
+	kvfree(out);
+	return ARRAY_SIZE(names);
+}
+
 static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 {
 	struct mlx5_ib_dev *dev;
@@ -2470,6 +2542,12 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 			(1ull << IB_USER_VERBS_CMD_DEALLOC_MW);
 	}
 
+	if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt) &&
+	    MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
+		dev->ib_dev.get_hw_stats	= mlx5_ib_get_hw_stats;
+		dev->ib_dev.alloc_hw_stats	= mlx5_ib_alloc_hw_stats;
+	}
+
 	if (MLX5_CAP_GEN(mdev, xrc)) {
 		dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd;
 		dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd;
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 2/2] IB/mlx5: Add port protocol stats

[Christoph Lameter]

Reviewed-by: Christoph Lameter <cl-vYTEC60ixJUAvxtiuMwx3w@public.gmane.org>

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 0/2] ConnectX-4/Connect-IB Port Counters

[Doug Ledford]

[-- Attachment #1: Type: text/plain, Size: 663 bytes --]

On 06/17/2016 08:10 AM, Leon Romanovsky wrote:
> This is supplementary part of mlx5 driver to generic IB counters infrastructure.
> 
> Mark Bloch (2):
>   IB/mlx5: Add per port counters
>   IB/mlx5: Add port protocol stats
> 
>  drivers/infiniband/hw/mlx5/main.c    | 134 ++++++++++++++++++++++++++++++++++-
>  drivers/infiniband/hw/mlx5/mlx5_ib.h |   6 ++
>  drivers/infiniband/hw/mlx5/qp.c      |   9 +++
>  3 files changed, 147 insertions(+), 2 deletions(-)
> 

Took a little work to make it apply, but series applied for 4.8, thanks.

-- 
Doug Ledford <dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
              GPG KeyID: 0E572FDD



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 884 bytes --]

Re: [PATCH 0/2] ConnectX-4/Connect-IB Port Counters

[Leon Romanovsky]

[-- Attachment #1: Type: text/plain, Size: 1044 bytes --]

On Thu, Jun 23, 2016 at 11:41:11AM -0400, Doug Ledford wrote:
> On 06/17/2016 08:10 AM, Leon Romanovsky wrote:
> > This is supplementary part of mlx5 driver to generic IB counters infrastructure.
> > 
> > Mark Bloch (2):
> >   IB/mlx5: Add per port counters
> >   IB/mlx5: Add port protocol stats
> > 
> >  drivers/infiniband/hw/mlx5/main.c    | 134 ++++++++++++++++++++++++++++++++++-
> >  drivers/infiniband/hw/mlx5/mlx5_ib.h |   6 ++
> >  drivers/infiniband/hw/mlx5/qp.c      |   9 +++
> >  3 files changed, 147 insertions(+), 2 deletions(-)
> > 
> 
> Took a little work to make it apply, but series applied for 4.8, thanks.

Thanks Doug,

I'm limited in my ability to predict the order of applies, and for
example this patchset applied cleanly for me.

For your reference, I have reference branch (testing/rdma-next) with all
topics in it.

Do you want to see it organized differently?

> 
> -- 
> Doug Ledford <dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
>               GPG KeyID: 0E572FDD
> 
> 



[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

Re: [PATCH 0/2] ConnectX-4/Connect-IB Port Counters

[Doug Ledford]

[-- Attachment #1: Type: text/plain, Size: 1335 bytes --]

On 06/23/2016 12:17 PM, Leon Romanovsky wrote:
> On Thu, Jun 23, 2016 at 11:41:11AM -0400, Doug Ledford wrote:
>> On 06/17/2016 08:10 AM, Leon Romanovsky wrote:
>>> This is supplementary part of mlx5 driver to generic IB counters infrastructure.
>>>
>>> Mark Bloch (2):
>>>   IB/mlx5: Add per port counters
>>>   IB/mlx5: Add port protocol stats
>>>
>>>  drivers/infiniband/hw/mlx5/main.c    | 134 ++++++++++++++++++++++++++++++++++-
>>>  drivers/infiniband/hw/mlx5/mlx5_ib.h |   6 ++
>>>  drivers/infiniband/hw/mlx5/qp.c      |   9 +++
>>>  3 files changed, 147 insertions(+), 2 deletions(-)
>>>
>>
>> Took a little work to make it apply, but series applied for 4.8, thanks.
> 
> Thanks Doug,
> 
> I'm limited in my ability to predict the order of applies, and for
> example this patchset applied cleanly for me.
> 
> For your reference, I have reference branch (testing/rdma-next) with all
> topics in it.
> 
> Do you want to see it organized differently?

No, I'm just making you aware of the fact that I had to fix things up so
that if you want to double check how I did the fix up, you can.  It
would be bad if I merged something and broke things subtly and no one
knew until much later.

-- 
Doug Ledford <dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
              GPG KeyID: 0E572FDD



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 884 bytes --]

Re: [PATCH 0/2] ConnectX-4/Connect-IB Port Counters

[Leon Romanovsky]

[-- Attachment #1: Type: text/plain, Size: 1515 bytes --]

On Thu, Jun 23, 2016 at 12:21:23PM -0400, Doug Ledford wrote:
> On 06/23/2016 12:17 PM, Leon Romanovsky wrote:
> > On Thu, Jun 23, 2016 at 11:41:11AM -0400, Doug Ledford wrote:
> >> On 06/17/2016 08:10 AM, Leon Romanovsky wrote:
> >>> This is supplementary part of mlx5 driver to generic IB counters infrastructure.
> >>>
> >>> Mark Bloch (2):
> >>>   IB/mlx5: Add per port counters
> >>>   IB/mlx5: Add port protocol stats
> >>>
> >>>  drivers/infiniband/hw/mlx5/main.c    | 134 ++++++++++++++++++++++++++++++++++-
> >>>  drivers/infiniband/hw/mlx5/mlx5_ib.h |   6 ++
> >>>  drivers/infiniband/hw/mlx5/qp.c      |   9 +++
> >>>  3 files changed, 147 insertions(+), 2 deletions(-)
> >>>
> >>
> >> Took a little work to make it apply, but series applied for 4.8, thanks.
> > 
> > Thanks Doug,
> > 
> > I'm limited in my ability to predict the order of applies, and for
> > example this patchset applied cleanly for me.
> > 
> > For your reference, I have reference branch (testing/rdma-next) with all
> > topics in it.
> > 
> > Do you want to see it organized differently?
> 
> No, I'm just making you aware of the fact that I had to fix things up so
> that if you want to double check how I did the fix up, you can.  It
> would be bad if I merged something and broke things subtly and no one
> knew until much later.

Excellent, we will do.

Thanks.

> 
> -- 
> Doug Ledford <dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
>               GPG KeyID: 0E572FDD
> 
> 



[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 819 bytes --]