[PATCH for-next v1 0/2] Advertise atomic operations support in mlx5

[PATCH for-next v1 0/2] Advertise atomic operations support in mlx5

[Eran Ben Elisha]

Hi Doug,

This patch set adds the functionality to advertise standard atomic operations
capabilities for mlx5 driver.  The Hardware can be configured to work in two
modes according to the device capabilities: 1. Big Endian requestor respond 2.
Host Endian requestor respond

If the firmware supports host endianness, try to configure the hardware to work
this way, and on success propagate this capability to the upper layers,
otherwise advertise that atomic operations aren't supported.

Thanks, Eran

Changes from v0:
	- Rewrite the commit message of the second patch in the series


Eran Ben Elisha (2):
  net/mlx5_core: Configure HW to support atomic request in host
    endianness
  IB/mlx5: Advertise atomic capabilities in query device

 drivers/infiniband/hw/mlx5/main.c              | 28 ++++++++++++-
 drivers/net/ethernet/mellanox/mlx5/core/main.c | 57 +++++++++++++++++++++++++-
 include/linux/mlx5/driver.h                    |  5 +++
 include/linux/mlx5/mlx5_ifc.h                  | 22 ++++++----
 4 files changed, 102 insertions(+), 10 deletions(-)

-- 
1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH for-next v1 1/2] net/mlx5_core: Configure HW to support atomic request in host endianness

[Eran Ben Elisha]

HW is capable of 2 requestor endianness modes for standard 8 Bytes
atomic: BE (0x0) and host endianness (0x1). Read the supported modes
from hca atomic capabilities and configure HW to host endianness mode if
supported.

Signed-off-by: Eran Ben Elisha <eranbe-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
Reviewed-by: Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---
 drivers/net/ethernet/mellanox/mlx5/core/main.c | 57 +++++++++++++++++++++++++-
 include/linux/mlx5/mlx5_ifc.h                  | 22 ++++++----
 2 files changed, 70 insertions(+), 9 deletions(-)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 03aabdd..682a4c0 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -73,6 +73,11 @@ struct mlx5_device_context {
 	void		       *context;
 };
 
+enum {
+	MLX5_ATOMIC_REQ_MODE_BE = 0x0,
+	MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1,
+};
+
 static struct mlx5_profile profile[] = {
 	[0] = {
 		.mask           = 0,
@@ -335,7 +340,7 @@ query_ex:
 	return err;
 }
 
-static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz)
+static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz, int opmod)
 {
 	u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)];
 	int err;
@@ -343,6 +348,7 @@ static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz)
 	memset(out, 0, sizeof(out));
 
 	MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP);
+	MLX5_SET(set_hca_cap_in, in, op_mod, opmod << 1);
 	err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out));
 	if (err)
 		return err;
@@ -352,6 +358,46 @@ static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz)
 	return err;
 }
 
+static int handle_hca_cap_atomic(struct mlx5_core_dev *dev)
+{
+	void *set_ctx;
+	void *set_hca_cap;
+	int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in);
+	int req_endianness;
+	int err;
+
+	if (MLX5_CAP_GEN(dev, atomic)) {
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC,
+					 HCA_CAP_OPMOD_GET_CUR);
+		if (err)
+			return err;
+	} else {
+		return 0;
+	}
+
+	req_endianness =
+		MLX5_CAP_ATOMIC(dev,
+				supported_atomic_req_8B_endianess_mode_1);
+
+	if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS)
+		return 0;
+
+	set_ctx = kzalloc(set_sz, GFP_KERNEL);
+	if (!set_ctx)
+		return -ENOMEM;
+
+	set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
+
+	/* Set requestor to host endianness */
+	MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianess_mode,
+		 MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS);
+
+	err = set_caps(dev, set_ctx, set_sz, MLX5_SET_HCA_CAP_OP_MOD_ATOMIC);
+
+	kfree(set_ctx);
+	return err;
+}
+
 static int handle_hca_cap(struct mlx5_core_dev *dev)
 {
 	void *set_ctx = NULL;
@@ -393,7 +439,8 @@ static int handle_hca_cap(struct mlx5_core_dev *dev)
 
 	MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12);
 
-	err = set_caps(dev, set_ctx, set_sz);
+	err = set_caps(dev, set_ctx, set_sz,
+		       MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
 
 query_ex:
 	kfree(set_ctx);
@@ -764,6 +811,12 @@ static int mlx5_dev_init(struct mlx5_core_dev *dev, struct pci_dev *pdev)
 		goto reclaim_boot_pages;
 	}
 
+	err = handle_hca_cap_atomic(dev);
+	if (err) {
+		dev_err(&pdev->dev, "handle_hca_cap_atomic failed\n");
+		goto reclaim_boot_pages;
+	}
+
 	err = mlx5_satisfy_startup_pages(dev, 0);
 	if (err) {
 		dev_err(&pdev->dev, "failed to allocate init pages\n");
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index dd20974..3da1951 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -67,6 +67,11 @@ enum {
 };
 
 enum {
+	MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE        = 0x0,
+	MLX5_SET_HCA_CAP_OP_MOD_ATOMIC                = 0x3,
+};
+
+enum {
 	MLX5_CMD_OP_QUERY_HCA_CAP                 = 0x100,
 	MLX5_CMD_OP_QUERY_ADAPTER                 = 0x101,
 	MLX5_CMD_OP_INIT_HCA                      = 0x102,
@@ -525,21 +530,24 @@ enum {
 struct mlx5_ifc_atomic_caps_bits {
 	u8         reserved_0[0x40];
 
-	u8         atomic_req_endianness[0x1];
-	u8         reserved_1[0x1f];
+	u8         atomic_req_8B_endianess_mode[0x2];
+	u8         reserved_1[0x4];
+	u8         supported_atomic_req_8B_endianess_mode_1[0x1];
 
-	u8         reserved_2[0x20];
+	u8         reserved_2[0x19];
 
-	u8         reserved_3[0x10];
-	u8         atomic_operations[0x10];
+	u8         reserved_3[0x20];
 
 	u8         reserved_4[0x10];
-	u8         atomic_size_qp[0x10];
+	u8         atomic_operations[0x10];
 
 	u8         reserved_5[0x10];
+	u8         atomic_size_qp[0x10];
+
+	u8         reserved_6[0x10];
 	u8         atomic_size_dc[0x10];
 
-	u8         reserved_6[0x720];
+	u8         reserved_7[0x720];
 };
 
 struct mlx5_ifc_odp_cap_bits {
-- 
1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH for-next v1 2/2] IB/mlx5: Advertise atomic capabilities in query device

[Eran Ben Elisha]

In order to ensure IB spec atomic correctness in atomic operations, if
HW is configured to host endianness, advertise IB_ATOMIC_HCA.  if not,
advertise IB_ATOMIC_NONE.

Signed-off-by: Eran Ben Elisha <eranbe-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
Reviewed-by: Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---
 drivers/infiniband/hw/mlx5/main.c | 28 +++++++++++++++++++++++++++-
 include/linux/mlx5/driver.h       |  5 +++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index bdd60a6..1139b67 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -63,6 +63,10 @@ static char mlx5_version[] =
 	DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"
 	DRIVER_VERSION " (" DRIVER_RELDATE ")\n";
 
+enum {
+	MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
+};
+
 static enum rdma_link_layer
 mlx5_ib_port_link_layer(struct ib_device *device)
 {
@@ -101,6 +105,28 @@ static int mlx5_get_vport_access_method(struct ib_device *ibdev)
 	return MLX5_VPORT_ACCESS_METHOD_HCA;
 }
 
+static void get_atomic_caps(struct mlx5_ib_dev *dev,
+			    struct ib_device_attr *props)
+{
+	u8 tmp;
+	u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
+	u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
+	u8 atomic_req_8B_endianness_mode =
+		MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianess_mode);
+
+	/* Check if HW supports 8 bytes standard atomic operations and capable
+	 * of host endianness respond
+	 */
+	tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
+	if (((atomic_operations & tmp) == tmp) &&
+	    (atomic_size_qp & MLX5_ATOMIC_SIZE_QP_8BYTES) &&
+	    (atomic_req_8B_endianness_mode)) {
+		props->atomic_cap = IB_ATOMIC_HCA;
+	} else {
+		props->atomic_cap = IB_ATOMIC_NONE;
+	}
+}
+
 static int mlx5_query_system_image_guid(struct ib_device *ibdev,
 					__be64 *sys_image_guid)
 {
@@ -286,7 +312,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	props->max_res_rd_atom	   = props->max_qp_rd_atom * props->max_qp;
 	props->max_srq_sge	   = max_rq_sg - 1;
 	props->max_fast_reg_page_list_len = (unsigned int)-1;
-	props->atomic_cap	   = IB_ATOMIC_NONE;
+	get_atomic_caps(dev, props);
 	props->masked_atomic_cap   = IB_ATOMIC_NONE;
 	props->max_mcast_grp	   = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
 	props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index 8b6d6f2..4c5a7e6 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -115,6 +115,11 @@ enum {
 	MLX5_REG_HOST_ENDIANNESS = 0x7004,
 };
 
+enum {
+	MLX5_ATOMIC_OPS_CMP_SWAP	= 1 << 0,
+	MLX5_ATOMIC_OPS_FETCH_ADD	= 1 << 1,
+};
+
 enum mlx5_page_fault_resume_flags {
 	MLX5_PAGE_FAULT_RESUME_REQUESTOR = 1 << 0,
 	MLX5_PAGE_FAULT_RESUME_WRITE	 = 1 << 1,
-- 
1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH for-next v1 0/2] Advertise atomic operations support in mlx5

[Doug Ledford]

[-- Attachment #1: Type: text/plain, Size: 1595 bytes --]

On 12/14/2015 09:34 AM, Eran Ben Elisha wrote:
> Hi Doug,
> 
> This patch set adds the functionality to advertise standard atomic operations
> capabilities for mlx5 driver.  The Hardware can be configured to work in two
> modes according to the device capabilities: 1. Big Endian requestor respond 2.
> Host Endian requestor respond
> 
> If the firmware supports host endianness, try to configure the hardware to work
> this way, and on success propagate this capability to the upper layers,
> otherwise advertise that atomic operations aren't supported.

If you are not going to allow any support except for host endian, then
why bother with all of the BE stuff, can't you just list it as supported
or not?

> Thanks, Eran
> 
> Changes from v0:
> 	- Rewrite the commit message of the second patch in the series
> 
> 
> Eran Ben Elisha (2):
>   net/mlx5_core: Configure HW to support atomic request in host
>     endianness

I can fix this up, but generally try to keep your subject of patches
short enough to fit in the generally accepted length.

>   IB/mlx5: Advertise atomic capabilities in query device
> 
>  drivers/infiniband/hw/mlx5/main.c              | 28 ++++++++++++-
>  drivers/net/ethernet/mellanox/mlx5/core/main.c | 57 +++++++++++++++++++++++++-
>  include/linux/mlx5/driver.h                    |  5 +++
>  include/linux/mlx5/mlx5_ifc.h                  | 22 ++++++----
>  4 files changed, 102 insertions(+), 10 deletions(-)
> 


-- 
Doug Ledford <dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
              GPG KeyID: 0E572FDD



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 884 bytes --]

Re: FW: [PATCH for-next v1 0/2] Advertise atomic operations support in mlx5

[eran ben elisha]

>>
>> If the firmware supports host endianness, try to configure the
>> hardware to work this way, and on success propagate this capability to
>> the upper layers, otherwise advertise that atomic operations aren't supported.
>
> If you are not going to allow any support except for host endian, then why bother with all of the BE stuff, can't you just list it as supported or not?

The HW default state is BE reply, need to change it's mode internally.
The user will never know about this, it do listed as supported/not.

>
>> Thanks, Eran
>>
>> Changes from v0:
>>       - Rewrite the commit message of the second patch in the series
>>
>>
>> Eran Ben Elisha (2):
>>   net/mlx5_core: Configure HW to support atomic request in host
>>     endianness
>
> I can fix this up, but generally try to keep your subject of patches short enough to fit in the generally accepted length.

Sure, will keep this in mind for next time.

>
>>   IB/mlx5: Advertise atomic capabilities in query device
>>
>>  drivers/infiniband/hw/mlx5/main.c              | 28 ++++++++++++-
>>  drivers/net/ethernet/mellanox/mlx5/core/main.c | 57 +++++++++++++++++++++++++-
>>  include/linux/mlx5/driver.h                    |  5 +++
>>  include/linux/mlx5/mlx5_ifc.h                  | 22 ++++++----
>>  4 files changed, 102 insertions(+), 10 deletions(-)
>>
>
>
> --
> Doug Ledford <dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
>               GPG KeyID: 0E572FDD
>
>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: FW: [PATCH for-next v1 0/2] Advertise atomic operations support in mlx5

[Doug Ledford]

[-- Attachment #1: Type: text/plain, Size: 1718 bytes --]

On 12/23/2015 12:05 PM, eran ben elisha wrote:
>>>
>>> If the firmware supports host endianness, try to configure the
>>> hardware to work this way, and on success propagate this capability to
>>> the upper layers, otherwise advertise that atomic operations aren't supported.
>>
>> If you are not going to allow any support except for host endian, then why bother with all of the BE stuff, can't you just list it as supported or not?
> 
> The HW default state is BE reply, need to change it's mode internally.
> The user will never know about this, it do listed as supported/not.

Ok, I've pulled this in.

>>
>>> Thanks, Eran
>>>
>>> Changes from v0:
>>>       - Rewrite the commit message of the second patch in the series
>>>
>>>
>>> Eran Ben Elisha (2):
>>>   net/mlx5_core: Configure HW to support atomic request in host
>>>     endianness
>>
>> I can fix this up, but generally try to keep your subject of patches short enough to fit in the generally accepted length.
> 
> Sure, will keep this in mind for next time.
> 
>>
>>>   IB/mlx5: Advertise atomic capabilities in query device
>>>
>>>  drivers/infiniband/hw/mlx5/main.c              | 28 ++++++++++++-
>>>  drivers/net/ethernet/mellanox/mlx5/core/main.c | 57 +++++++++++++++++++++++++-
>>>  include/linux/mlx5/driver.h                    |  5 +++
>>>  include/linux/mlx5/mlx5_ifc.h                  | 22 ++++++----
>>>  4 files changed, 102 insertions(+), 10 deletions(-)
>>>
>>
>>
>> --
>> Doug Ledford <dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
>>               GPG KeyID: 0E572FDD
>>
>>


-- 
Doug Ledford <dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
              GPG KeyID: 0E572FDD



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 884 bytes --]