Re: [RFC PATCH 1/5] IB/core: Add Core Capability flags to ib_device

Doug Ledford
2015-05-04T14:41:32Z

[-- Attachment #1: Type: text/plain, Size: 4864 bytes --]

On Mon, 2015-05-04 at 02:14 -0400, ira.weiny-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org wrote:
> From: Ira Weiny <ira.weiny-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
>
> Add Core capability flags to each port attribute and read those into ib_device
> upon registration for each port.
>
> Signed-off-by: Ira Weiny <ira.weiny-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> ---
> drivers/infiniband/core/device.c | 41 ++++++++++++++++++++++++++++++++++++++
> include/rdma/ib_verbs.h | 22 ++++++++++++++++++++
> 2 files changed, 63 insertions(+), 0 deletions(-)
>
> diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
> index b360350..6a37255 100644
> --- a/drivers/infiniband/core/device.c
> +++ b/drivers/infiniband/core/device.c
> @@ -262,6 +262,37 @@ out:
> return ret;
> }
>
> +static int read_core_cap_flags(struct ib_device *device)
> +{
> + struct ib_port_attr tprops;
> + int num_ports, ret = -ENOMEM;
> + u8 port_index;
> +
> + num_ports = device->phys_port_cnt;
> +
> + device->core_cap_flags = kzalloc(sizeof(*device->core_cap_flags)
> + * (num_ports+1),
> + GFP_KERNEL);
> + if (!device->core_cap_flags)
> + return -ENOMEM;
> +
> + for (port_index = 0; port_index <= num_ports; ++port_index) {
> + if ((port_index == 0 && device->node_type != RDMA_NODE_IB_SWITCH))
> + continue;
> +
> + ret = ib_query_port(device, port_index, &tprops);
> + if (ret)
> + goto err;
> +
> + device->core_cap_flags[port_index] = tprops.core_cap_flags;
> + }
> +
> + return 0;
> +err:
> + kfree(device->core_cap_flags);
> + return ret;
> +}
> +
> /**
> * ib_register_device - Register an IB device with IB core
> * @device:Device to register
> @@ -302,12 +333,21 @@ int ib_register_device(struct ib_device *device,
> goto out;
> }
>
> + ret = read_core_cap_flags(device);
> + if (ret) {
> + dev_err(&device->dev, "Couldn't create Core Capability flags\n");
> + kfree(device->gid_tbl_len);
> + kfree(device->pkey_tbl_len);
> + goto out;
> + }
> +
> ret = ib_device_register_sysfs(device, port_callback);
> if (ret) {
> printk(KERN_WARNING "Couldn't register device %s with driver model\n",
> device->name);
> kfree(device->gid_tbl_len);
> kfree(device->pkey_tbl_len);
> + kfree(device->core_cap_flags);
> goto out;
> }
>
> @@ -351,6 +391,7 @@ void ib_unregister_device(struct ib_device *device)
>
> kfree(device->gid_tbl_len);
> kfree(device->pkey_tbl_len);
> + kfree(device->core_cap_flags);
>
> mutex_unlock(&device_mutex);
>
> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
> index c724114..4de2758 100644
> --- a/include/rdma/ib_verbs.h
> +++ b/include/rdma/ib_verbs.h
> @@ -353,11 +353,32 @@ union rdma_protocol_stats {
> struct iw_protocol_stats iw;
> };
>
> +/* Define bits for the various functionality this port needs to be supported by
> + * the core.
> + */
> +/* Management 0x00000000FFFFFFFF */
> +#define RDMA_CORE_CAP_IB_MAD 0x0000000000000001ULL
> +#define RDMA_CORE_CAP_IB_SMI 0x0000000000000002ULL
> +#define RDMA_CORE_CAP_IB_CM 0x0000000000000004ULL
> +#define RDMA_CORE_CAP_IW_CM 0x0000000000000008ULL
> +#define RDMA_CORE_CAP_IB_SA 0x0000000000000010ULL
> +
> +/* Address format 0x0000FFFF00000000 */
> +#define RDMA_CORE_CAP_AF_IB 0x0000000100000000ULL
> +#define RDMA_CORE_CAP_ETH_AH 0x0000000200000000ULL
> +
> +/* Protocol 0xFFFF000000000000 */
> +#define RDMA_CORE_CAP_PROT_IB 0x0001000000000000ULL
> +#define RDMA_CORE_CAP_PROT_IBOE 0x0002000000000000ULL
> +#define RDMA_CORE_CAP_PROT_IWARP 0x0004000000000000ULL
> +#define RDMA_CORE_CAP_PROT_USNIC_UDP 0x0008000000000000ULL

In accordance with what we've been talking about, drop IBOE for ROCE.

Drop the UDP off of USNIC, then define a bit for CAP_PROT_UDP_ENCAP.
USNIC will be just USNIC, USNIC_UDP will be USNIC | UDP_ENCAP, ROCE v1
will be ROCE, and ROCEv2 will be ROCE | UDP_ENCAP.

> +
> struct ib_port_attr {
> enum ib_port_state state;
> enum ib_mtu max_mtu;
> enum ib_mtu active_mtu;
> int gid_tbl_len;
> + u64 core_cap_flags;

I think u32 should be enough here, and will help keep our footprint
smaller.

> u32 port_cap_flags;
> u32 max_msg_sz;
> u32 bad_pkey_cntr;
> @@ -1684,6 +1705,7 @@ struct ib_device {
> u32 local_dma_lkey;
> u8 node_type;
> u8 phys_port_cnt;
> + u64 *core_cap_flags; /* Per port core capability flags */

Ditto.

> };
>
> struct ib_client {

--
Doug Ledford <dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
GPG KeyID: 0E572FDD

[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 819 bytes --]