Re: [PATCH 09/30] IB/core: Modify ib_verbs and cma in order to use roce_gid_cache

Matan Barak
2015-02-23T10:17:10Z

On 2/23/2015 7:25 AM, Devesh Sharma wrote:
> Hi Matan,
>
> Please find a comment inline below:
>
> -Regards
> Devesh
>> -----Original Message-----
>> From: linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org [mailto:linux-rdma-
>> owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org] On Behalf Of Somnath Kotur
>> Sent: Friday, February 20, 2015 3:32 AM
>> To: roland-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org
>> Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org; Matan Barak; Somnath Kotur
>> Subject: [PATCH 09/30] IB/core: Modify ib_verbs and cma in order to use
>> roce_gid_cache
>>
>> From: Matan Barak <matanb-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
>>
>> Previously, we resolved the dmac and took the smac and vlan from the resolved
>> address. Changing that into finding a net device that matches the IP and vlan of
>> the network packet and querying the RoCE GID cache for this net device, GID
>> and GID type.
>>
>> ocrdma driver changes were done by Somnath Kotur
>> <Somnath.Kotur-iH1Dq9VlAzfQT0dZR+AlfA@public.gmane.org>
>>
>> Signed-off-by: Matan Barak <matanb-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
>> Signed-off-by: Somnath Kotur <somnath.kotur-laKkSmNT4hbQT0dZR+AlfA@public.gmane.org>
>> ---
>> drivers/infiniband/core/addr.c | 3 +-
>> drivers/infiniband/core/cm.c | 30 ------
>> drivers/infiniband/core/cma.c | 9 --
>> drivers/infiniband/core/core_priv.h | 4 +-
>> drivers/infiniband/core/sa_query.c | 4 -
>> drivers/infiniband/core/ucma.c | 1 -
>> drivers/infiniband/core/uverbs_cmd.c | 6 +-
>> drivers/infiniband/core/verbs.c | 159 +++++++++++++++++------------
>> drivers/infiniband/hw/mlx4/ah.c | 15 +++-
>> drivers/infiniband/hw/mlx4/mad.c | 12 ++-
>> drivers/infiniband/hw/mlx4/mcg.c | 2 +-
>> drivers/infiniband/hw/mlx4/mlx4_ib.h | 2 +-
>> drivers/infiniband/hw/mlx4/qp.c | 42 ++++++--
>> drivers/infiniband/hw/ocrdma/ocrdma.h | 1 +
>> drivers/infiniband/hw/ocrdma/ocrdma_ah.c | 20 +++--
>> drivers/infiniband/hw/ocrdma/ocrdma_hw.c | 17 ++-
>> include/rdma/ib_addr.h | 2 +-
>> include/rdma/ib_sa.h | 2 -
>> include/rdma/ib_verbs.h | 7 +-
>> 19 files changed, 183 insertions(+), 155 deletions(-)
>>
>> diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
>> index f80da50..43af7f5 100644
>> --- a/drivers/infiniband/core/addr.c
>> +++ b/drivers/infiniband/core/addr.c
>> @@ -458,7 +458,7 @@ static void resolve_cb(int status, struct sockaddr
>> *src_addr, }
>>
>> int rdma_addr_find_dmac_by_grh(union ib_gid *sgid, union ib_gid *dgid, u8
>> *dmac,
>> - u16 *vlan_id)
>> + u16 *vlan_id, int if_index)
>> {
>> int ret = 0;
>> struct rdma_dev_addr dev_addr;
>> @@ -481,6 +481,7 @@ int rdma_addr_find_dmac_by_grh(union ib_gid *sgid,
>> union ib_gid *dgid, u8 *dmac,
>> return ret;
>>
>> memset(&dev_addr, 0, sizeof(dev_addr));
>> + dev_addr.bound_dev_if = if_index;
>>
>> ctx.addr = &dev_addr;
>> init_completion(&ctx.comp);
>> diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index
>> d88f2ae..7974e74 100644
>> --- a/drivers/infiniband/core/cm.c
>> +++ b/drivers/infiniband/core/cm.c
>> @@ -178,8 +178,6 @@ struct cm_av {
>> struct ib_ah_attr ah_attr;
>> u16 pkey_index;
>> u8 timeout;
>> - u8 valid;
>> - u8 smac[ETH_ALEN];
>> };
>>
>> struct cm_work {
>> @@ -382,7 +380,6 @@ static int cm_init_av_by_path(struct ib_sa_path_rec
>> *path, struct cm_av *av)
>> &av->ah_attr);
>> av->timeout = path->packet_life_time + 1;
>>
>> - av->valid = 1;
>> return 0;
>> }
>>
>> @@ -1563,7 +1560,6 @@ static int cm_req_handler(struct cm_work *work)
>> cm_format_paths_from_req(req_msg, &work->path[0], &work-
>>> path[1]);
>>
>> memcpy(work->path[0].dmac, cm_id_priv->av.ah_attr.dmac,
>> ETH_ALEN);
>> - work->path[0].vlan_id = cm_id_priv->av.ah_attr.vlan_id;
>> ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av);
>> if (ret) {
>> ib_get_cached_gid(work->port->cm_dev->ib_device,
>> @@ -3511,32 +3507,6 @@ static int cm_init_qp_rtr_attr(struct cm_id_private
>> *cm_id_priv,
>> *qp_attr_mask = IB_QP_STATE | IB_QP_AV |
>> IB_QP_PATH_MTU |
>> IB_QP_DEST_QPN | IB_QP_RQ_PSN;
>> qp_attr->ah_attr = cm_id_priv->av.ah_attr;
>> - if (!cm_id_priv->av.valid) {
>> - spin_unlock_irqrestore(&cm_id_priv->lock, flags);
>> - return -EINVAL;
>> - }
>> - if (cm_id_priv->av.ah_attr.vlan_id != 0xffff) {
>> - qp_attr->vlan_id = cm_id_priv->av.ah_attr.vlan_id;
>> - *qp_attr_mask |= IB_QP_VID;
>> - }
>> - if (!is_zero_ether_addr(cm_id_priv->av.smac)) {
>> - memcpy(qp_attr->smac, cm_id_priv->av.smac,
>> - sizeof(qp_attr->smac));
>> - *qp_attr_mask |= IB_QP_SMAC;
>> - }
>> - if (cm_id_priv->alt_av.valid) {
>> - if (cm_id_priv->alt_av.ah_attr.vlan_id != 0xffff) {
>> - qp_attr->alt_vlan_id =
>> - cm_id_priv->alt_av.ah_attr.vlan_id;
>> - *qp_attr_mask |= IB_QP_ALT_VID;
>> - }
>> - if (!is_zero_ether_addr(cm_id_priv->alt_av.smac)) {
>> - memcpy(qp_attr->alt_smac,
>> - cm_id_priv->alt_av.smac,
>> - sizeof(qp_attr->alt_smac));
>> - *qp_attr_mask |= IB_QP_ALT_SMAC;
>> - }
>> - }
>> qp_attr->path_mtu = cm_id_priv->path_mtu;
>> qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv-
>>> remote_qpn);
>> qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn); diff --git
>> a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index
>> 335def9..659676c 100644
>> --- a/drivers/infiniband/core/cma.c
>> +++ b/drivers/infiniband/core/cma.c
>> @@ -666,15 +666,6 @@ static int cma_modify_qp_rtr(struct rdma_id_private
>> *id_priv,
>> if (ret)
>> goto out;
>>
>> - if (rdma_node_get_transport(id_priv->cma_dev->device->node_type)
>> - == RDMA_TRANSPORT_IB &&
>> - rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)
>> - == IB_LINK_LAYER_ETHERNET) {
>> - ret = rdma_addr_find_smac_by_sgid(&sgid, qp_attr.smac,
>> NULL);
>> -
>> - if (ret)
>> - goto out;
>> - }
>> if (conn_param)
>> qp_attr.max_dest_rd_atomic = conn_param-
>>> responder_resources;
>> ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); diff --git
>> a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
>> index d6e73f8..fbe5922 100644
>> --- a/drivers/infiniband/core/core_priv.h
>> +++ b/drivers/infiniband/core/core_priv.h
>> @@ -52,8 +52,8 @@ void ib_sysfs_cleanup(void); int ib_cache_setup(void);
>> void ib_cache_cleanup(void);
>>
>> -int ib_resolve_eth_l2_attrs(struct ib_qp *qp,
>> - struct ib_qp_attr *qp_attr, int *qp_attr_mask);
>> +int ib_resolve_eth_dmac(struct ib_qp *qp,
>> + struct ib_qp_attr *qp_attr, int *qp_attr_mask);
>>
>> typedef void (*roce_netdev_callback)(struct ib_device *device, u8 port,
>> struct net_device *idev, void *cookie); diff --git
>> a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c
>> index 5b20237..705b6b8 100644
>> --- a/drivers/infiniband/core/sa_query.c
>> +++ b/drivers/infiniband/core/sa_query.c
>> @@ -559,11 +559,7 @@ int ib_init_ah_from_path(struct ib_device *device, u8
>> port_num,
>> }
>> if (force_grh) {
>> memcpy(ah_attr->dmac, rec->dmac, ETH_ALEN);
>> - ah_attr->vlan_id = rec->vlan_id;
>> - } else {
>> - ah_attr->vlan_id = 0xffff;
>> }
>> -
>> return 0;
>> }
>> EXPORT_SYMBOL(ib_init_ah_from_path);
>> diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c
>> index 45d67e9..5eacda4 100644
>> --- a/drivers/infiniband/core/ucma.c
>> +++ b/drivers/infiniband/core/ucma.c
>> @@ -1125,7 +1125,6 @@ static int ucma_set_ib_path(struct ucma_context
>> *ctx,
>> return -EINVAL;
>>
>> memset(&sa_path, 0, sizeof(sa_path));
>> - sa_path.vlan_id = 0xffff;
>>
>> ib_sa_unpack_path(path_data->path_rec, &sa_path);
>> ret = rdma_set_ib_paths(ctx->cm_id, &sa_path, 1); diff --git
>> a/drivers/infiniband/core/uverbs_cmd.c
>> b/drivers/infiniband/core/uverbs_cmd.c
>> index b7943ff..07d7f13 100644
>> --- a/drivers/infiniband/core/uverbs_cmd.c
>> +++ b/drivers/infiniband/core/uverbs_cmd.c
>> @@ -2089,15 +2089,16 @@ ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file
>> *file,
>> attr->alt_ah_attr.port_num = cmd.alt_dest.port_num;
>>
>> if (qp->real_qp == qp) {
>> - ret = ib_resolve_eth_l2_attrs(qp, attr, &cmd.attr_mask);
>> + ret = ib_resolve_eth_dmac(qp, attr, &cmd.attr_mask);
>> if (ret)
>> - goto out;
>> + goto out_put;
>> ret = qp->device->modify_qp(qp, attr,
>> modify_qp_mask(qp->qp_type, cmd.attr_mask),
>> &udata);
>> } else {
>> ret = ib_modify_qp(qp, attr, modify_qp_mask(qp->qp_type,
>> cmd.attr_mask));
>> }
>>
>> +out_put:
>> put_qp_read(qp);
>>
>> if (ret)
>> @@ -2552,7 +2553,6 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file
>> *file,
>> attr.grh.sgid_index = cmd.attr.grh.sgid_index;
>> attr.grh.hop_limit = cmd.attr.grh.hop_limit;
>> attr.grh.traffic_class = cmd.attr.grh.traffic_class;
>> - attr.vlan_id = 0;
>> memset(&attr.dmac, 0, sizeof(attr.dmac));
>> memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16);
>>
>> diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
>> index 1fe3e71..2c54d31 100644
>> --- a/drivers/infiniband/core/verbs.c
>> +++ b/drivers/infiniband/core/verbs.c
>> @@ -41,6 +41,9 @@
>> #include <linux/export.h>
>> #include <linux/string.h>
>> #include <linux/slab.h>
>> +#include <linux/in.h>
>> +#include <linux/in6.h>
>> +#include <net/addrconf.h>
>>
>> #include <rdma/ib_verbs.h>
>> #include <rdma/ib_cache.h>
>> @@ -192,6 +195,35 @@ struct ib_ah *ib_create_ah(struct ib_pd *pd, struct
>> ib_ah_attr *ah_attr) } EXPORT_SYMBOL(ib_create_ah);
>>
>> +struct find_gid_index_context {
>> + u16 vlan_id;
>> +};
>> +
>> +static bool find_gid_index(const union ib_gid *gid,
>> + const struct ib_gid_attr *gid_attr,
>> + void *context)
>> +{
>> + struct find_gid_index_context *ctx =
>> + (struct find_gid_index_context *)context;
>> +
>> + if ((!!(ctx->vlan_id != 0xffff) == !is_vlan_dev(gid_attr->ndev)) ||
>> + (is_vlan_dev(gid_attr->ndev) &&
>> + vlan_dev_vlan_id(gid_attr->ndev) != ctx->vlan_id))
>> + return false;
>> +
>> + return true;
>> +}
>> +
>> +static int get_sgid_index_from_eth(struct ib_device *device, u8 port_num,
>> + u16 vlan_id, union ib_gid *sgid,
>> + u16 *gid_index)
>> +{
>> + struct find_gid_index_context context = {.vlan_id = vlan_id};
>> +
>> + return ib_find_gid_by_filter(device, sgid, port_num, find_gid_index,
>> + &context, gid_index);
>> +}
>> +
>> int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc
>> *wc,
>> struct ib_grh *grh, struct ib_ah_attr *ah_attr) { @@ -
>> 203,21 +235,30 @@ int ib_init_ah_from_wc(struct ib_device *device, u8
>> port_num, struct ib_wc *wc,
>>
>> memset(ah_attr, 0, sizeof *ah_attr);
>> if (is_eth) {
>> + u16 vlan_id = wc->wc_flags & IB_WC_WITH_VLAN ?
>> + wc->vlan_id : 0xffff;
>> +
>> if (!(wc->wc_flags & IB_WC_GRH))
>> return -EPROTOTYPE;
>>
>> - if (wc->wc_flags & IB_WC_WITH_SMAC &&
>> - wc->wc_flags & IB_WC_WITH_VLAN) {
>> - memcpy(ah_attr->dmac, wc->smac, ETH_ALEN);
>> - ah_attr->vlan_id = wc->vlan_id;
>> - } else {
>> + if (!(wc->wc_flags & IB_WC_WITH_SMAC) ||
>> + !(wc->wc_flags & IB_WC_WITH_VLAN)) {
>> ret = rdma_addr_find_dmac_by_grh(&grh->dgid,
>> &grh->sgid,
>> - ah_attr->dmac, &ah_attr->vlan_id);
>> + ah_attr->dmac,
>> + wc->wc_flags &
>> IB_WC_WITH_VLAN ?
>> + NULL : &vlan_id,
>> + 0);
>> if (ret)
>> return ret;
>> }
>> - } else {
>> - ah_attr->vlan_id = 0xffff;
>> +
>> + ret = get_sgid_index_from_eth(device, port_num, vlan_id,
>> + &grh->dgid, &gid_index);
>> + if (ret)
>> + return ret;
>> +
>> + if (wc->wc_flags & IB_WC_WITH_SMAC)
>> + memcpy(ah_attr->dmac, wc->smac, ETH_ALEN);
>> }
>>
>> ah_attr->dlid = wc->slid;
>> @@ -229,10 +270,14 @@ int ib_init_ah_from_wc(struct ib_device *device, u8
>> port_num, struct ib_wc *wc,
>> ah_attr->ah_flags = IB_AH_GRH;
>> ah_attr->grh.dgid = grh->sgid;
>>
>> - ret = ib_find_cached_gid(device, &grh->dgid, IB_GID_TYPE_IB,
>> - NULL, 0, &port_num, &gid_index);
>> - if (ret)
>> - return ret;
>> + if (!is_eth) {
>> + ret = ib_find_cached_gid_by_port(device, &grh->dgid,
>> + IB_GID_TYPE_IB,
>> + port_num, NULL, 0,
>> + &gid_index);
>> + if (ret)
>> + return ret;
>> + }
>>
>> ah_attr->grh.sgid_index = (u8) gid_index;
>> flow_class = be32_to_cpu(grh->version_tclass_flow);
>> @@ -502,9 +547,7 @@ EXPORT_SYMBOL(ib_create_qp); static const struct {
>> int valid;
>> enum ib_qp_attr_mask req_param[IB_QPT_MAX];
>> - enum ib_qp_attr_mask req_param_add_eth[IB_QPT_MAX];
>> enum ib_qp_attr_mask opt_param[IB_QPT_MAX];
>> - enum ib_qp_attr_mask opt_param_add_eth[IB_QPT_MAX];
>> } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
>> [IB_QPS_RESET] = {
>> [IB_QPS_RESET] = { .valid = 1 },
>> @@ -585,12 +628,6 @@ static const struct {
>>
>> IB_QP_MAX_DEST_RD_ATOMIC |
>> IB_QP_MIN_RNR_TIMER),
>> },
>> - .req_param_add_eth = {
>> - [IB_QPT_RC] = (IB_QP_SMAC),
>> - [IB_QPT_UC] = (IB_QP_SMAC),
>> - [IB_QPT_XRC_INI] = (IB_QP_SMAC),
>> - [IB_QPT_XRC_TGT] = (IB_QP_SMAC)
>> - },
>> .opt_param = {
>> [IB_QPT_UD] = (IB_QP_PKEY_INDEX
>> |
>> IB_QP_QKEY),
>> @@ -611,21 +648,7 @@ static const struct {
>> [IB_QPT_GSI] = (IB_QP_PKEY_INDEX
>> |
>> IB_QP_QKEY),
>> },
>> - .opt_param_add_eth = {
>> - [IB_QPT_RC] = (IB_QP_ALT_SMAC
>> |
>> - IB_QP_VID
>> |
>> - IB_QP_ALT_VID),
>> - [IB_QPT_UC] = (IB_QP_ALT_SMAC
>> |
>> - IB_QP_VID
>> |
>> - IB_QP_ALT_VID),
>> - [IB_QPT_XRC_INI] = (IB_QP_ALT_SMAC
>> |
>> - IB_QP_VID
>> |
>> - IB_QP_ALT_VID),
>> - [IB_QPT_XRC_TGT] = (IB_QP_ALT_SMAC
>> |
>> - IB_QP_VID
>> |
>> - IB_QP_ALT_VID)
>> - }
>> - }
>> + },
>> },
>> [IB_QPS_RTR] = {
>> [IB_QPS_RESET] = { .valid = 1 },
>> @@ -847,13 +870,6 @@ int ib_modify_qp_is_ok(enum ib_qp_state cur_state,
>> enum ib_qp_state next_state,
>> req_param = qp_state_table[cur_state][next_state].req_param[type];
>> opt_param = qp_state_table[cur_state][next_state].opt_param[type];
>>
>> - if (ll == IB_LINK_LAYER_ETHERNET) {
>> - req_param |= qp_state_table[cur_state][next_state].
>> - req_param_add_eth[type];
>> - opt_param |= qp_state_table[cur_state][next_state].
>> - opt_param_add_eth[type];
>> - }
>> -
>> if ((mask & req_param) != req_param)
>> return 0;
>>
>> @@ -864,41 +880,52 @@ int ib_modify_qp_is_ok(enum ib_qp_state cur_state,
>> enum ib_qp_state next_state, } EXPORT_SYMBOL(ib_modify_qp_is_ok);
>>
>> -int ib_resolve_eth_l2_attrs(struct ib_qp *qp,
>> - struct ib_qp_attr *qp_attr, int *qp_attr_mask)
>> +int ib_resolve_eth_dmac(struct ib_qp *qp,
>> + struct ib_qp_attr *qp_attr, int *qp_attr_mask)
>> {
>> int ret = 0;
>> - union ib_gid sgid;
>>
>> if ((*qp_attr_mask & IB_QP_AV) &&
>> - (rdma_port_get_link_layer(qp->device, qp_attr->ah_attr.port_num)
>> == IB_LINK_LAYER_ETHERNET)) {
>> - ret = ib_query_gid(qp->device, qp_attr->ah_attr.port_num,
>> - qp_attr->ah_attr.grh.sgid_index, &sgid,
>> - NULL);
>> - if (ret)
>> - goto out;
>> + (rdma_port_get_link_layer(qp->device, qp_attr->ah_attr.port_num)
>> ==
>> + IB_LINK_LAYER_ETHERNET)) {
>> if (rdma_link_local_addr((struct in6_addr *)qp_attr-
>>> ah_attr.grh.dgid.raw)) {
>> - rdma_get_ll_mac((struct in6_addr *)qp_attr-
>>> ah_attr.grh.dgid.raw, qp_attr->ah_attr.dmac);
>> - rdma_get_ll_mac((struct in6_addr *)sgid.raw, qp_attr-
>>> smac);
>> - if (!(*qp_attr_mask & IB_QP_VID))
>> - qp_attr->vlan_id = rdma_get_vlan_id(&sgid);
>> + rdma_get_ll_mac((struct in6_addr *)qp_attr-
>>> ah_attr.grh.dgid.raw,
>> + qp_attr->ah_attr.dmac);
>> } else {
>> - ret = rdma_addr_find_dmac_by_grh(&sgid, &qp_attr-
>>> ah_attr.grh.dgid,
>> - qp_attr->ah_attr.dmac, &qp_attr-
>>> vlan_id);
>> - if (ret)
>> - goto out;
>> - ret = rdma_addr_find_smac_by_sgid(&sgid, qp_attr-
>>> smac, NULL);
>> - if (ret)
>> + union ib_gid sgid;
>> + struct ib_gid_attr sgid_attr;
>> + int ifindex;
>> +
>> + rcu_read_lock();
>> + ret = ib_query_gid(qp->device,
>> + qp_attr->ah_attr.port_num,
>> + qp_attr->ah_attr.grh.sgid_index,
>> + &sgid, &sgid_attr);
>> +
>> + if (ret || !sgid_attr.ndev) {
>> + if (!ret)
>> + ret = -ENXIO;
>> + rcu_read_unlock();
>> goto out;
>> + }
>> +
>> + dev_hold(sgid_attr.ndev);
>> + ifindex = sgid_attr.ndev->ifindex;
>> +
>> + rcu_read_unlock();
>> +
>> + ret = rdma_addr_find_dmac_by_grh(&sgid,
>> + &qp_attr-
>>> ah_attr.grh.dgid,
>> + qp_attr-
>>> ah_attr.dmac,
>> + NULL, ifindex);
>
> Vlan-ID can also be resolved here and passed to vendor specific modify_qp?
>
> Similarly for UD:
> ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
> const char __user *buf, int in_len,
> int out_len)
>
> could resolve the dmac and vlan-id before calling ib_create_ah() in uverbs_cmd.c
>
> these changes would make vendor drivers independent of how the attributes are resolved.
>

Hi,

The problem with this approach is that some vendors don't go through
ib_uverbs_create_ah. Moving the resolution code to this function might
break user-space applications.

Regards,
Matan

>> +
>> + dev_put(sgid_attr.ndev);
>> }
>> - *qp_attr_mask |= IB_QP_SMAC;
>> - if (qp_attr->vlan_id < 0xFFFF)
>> - *qp_attr_mask |= IB_QP_VID;
>> }
>> out:
>> return ret;
>> }
>> -EXPORT_SYMBOL(ib_resolve_eth_l2_attrs);
>> +EXPORT_SYMBOL(ib_resolve_eth_dmac);
>>
>>
>> int ib_modify_qp(struct ib_qp *qp,
>> @@ -907,7 +934,7 @@ int ib_modify_qp(struct ib_qp *qp, {
>> int ret;
>>
>> - ret = ib_resolve_eth_l2_attrs(qp, qp_attr, &qp_attr_mask);
>> + ret = ib_resolve_eth_dmac(qp, qp_attr, &qp_attr_mask);
>> if (ret)
>> return ret;
>>
>> diff --git a/drivers/infiniband/hw/mlx4/ah.c b/drivers/infiniband/hw/mlx4/ah.c
>> index f50a546..aaeeb60 100644
>> --- a/drivers/infiniband/hw/mlx4/ah.c
>> +++ b/drivers/infiniband/hw/mlx4/ah.c
>> @@ -76,7 +76,9 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd,
>> struct ib_ah_attr *ah_attr
>> struct mlx4_dev *dev = ibdev->dev;
>> int is_mcast = 0;
>> struct in6_addr in6;
>> - u16 vlan_tag;
>> + u16 vlan_tag = 0xffff;
>> + union ib_gid sgid;
>> + struct ib_gid_attr gid_attr;
>>
>> memcpy(&in6, ah_attr->grh.dgid.raw, sizeof(in6));
>> if (rdma_is_multicast_addr(&in6)) {
>> @@ -85,7 +87,16 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd,
>> struct ib_ah_attr *ah_attr
>> } else {
>> memcpy(ah->av.eth.mac, ah_attr->dmac, ETH_ALEN);
>> }
>> - vlan_tag = ah_attr->vlan_id;
>> + rcu_read_lock();
>> + ib_get_cached_gid(pd->device, ah_attr->port_num,
>> + ah_attr->grh.sgid_index, &sgid, &gid_attr);
>> + memset(ah->av.eth.s_mac, 0, ETH_ALEN);
>> + if (gid_attr.ndev) {
>> + if (is_vlan_dev(gid_attr.ndev))
>> + vlan_tag = vlan_dev_vlan_id(gid_attr.ndev);
>> + memcpy(ah->av.eth.s_mac, gid_attr.ndev->dev_addr,
>> ETH_ALEN);
>> + }
>> + rcu_read_unlock();
>> if (vlan_tag < 0x1000)
>> vlan_tag |= (ah_attr->sl & 7) << 13;
>> ah->av.eth.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr-
>>> port_num << 24)); diff --git a/drivers/infiniband/hw/mlx4/mad.c
>> b/drivers/infiniband/hw/mlx4/mad.c
>> index 82a7dd8..e686e95 100644
>> --- a/drivers/infiniband/hw/mlx4/mad.c
>> +++ b/drivers/infiniband/hw/mlx4/mad.c
>> @@ -1154,7 +1154,7 @@ static int is_proxy_qp0(struct mlx4_ib_dev *dev, int
>> qpn, int slave) int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8
>> port,
>> enum ib_qp_type dest_qpt, u16 pkey_index,
>> u32 remote_qpn, u32 qkey, struct ib_ah_attr *attr,
>> - u8 *s_mac, struct ib_mad *mad)
>> + u8 *s_mac, u16 vlan_id, struct ib_mad *mad)
>> {
>> struct ib_sge list;
>> struct ib_send_wr wr, *bad_wr;
>> @@ -1241,6 +1241,9 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev,
>> int slave, u8 port,
>> wr.send_flags = IB_SEND_SIGNALED;
>> if (s_mac)
>> memcpy(to_mah(ah)->av.eth.s_mac, s_mac, 6);
>> + if (vlan_id < 0x1000)
>> + vlan_id |= (attr->sl & 7) << 13;
>> + to_mah(ah)->av.eth.vlan = cpu_to_be16(vlan_id);
>>
>>
>> ret = ib_post_send(send_qp, &wr, &bad_wr); @@ -1277,6 +1280,7 @@
>> static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct
>> ib_wc
>> u8 *slave_id;
>> int slave;
>> int port;
>> + u16 vlan_id;
>>
>> /* Get slave that sent this packet */
>> if (wc->src_qp < dev->dev->phys_caps.base_proxy_sqpn || @@ -
>> 1362,10 +1366,10 @@ static void mlx4_ib_multiplex_mad(struct
>> mlx4_ib_demux_pv_ctx *ctx, struct ib_wc
>> return;
>> ah_attr.port_num = port;
>> memcpy(ah_attr.dmac, tunnel->hdr.mac, 6);
>> - ah_attr.vlan_id = be16_to_cpu(tunnel->hdr.vlan);
>> + vlan_id = be16_to_cpu(tunnel->hdr.vlan);
>> /* if slave have default vlan use it */
>> mlx4_get_slave_default_vlan(dev->dev, ctx->port, slave,
>> - &ah_attr.vlan_id, &ah_attr.sl);
>> + &vlan_id, &ah_attr.sl);
>>
>> mlx4_ib_send_to_wire(dev, slave, ctx->port,
>> is_proxy_qp0(dev, wc->src_qp, slave) ?
>> @@ -1373,7 +1377,7 @@ static void mlx4_ib_multiplex_mad(struct
>> mlx4_ib_demux_pv_ctx *ctx, struct ib_wc
>> be16_to_cpu(tunnel->hdr.pkey_index),
>> be32_to_cpu(tunnel->hdr.remote_qpn),
>> be32_to_cpu(tunnel->hdr.qkey),
>> - &ah_attr, wc->smac, &tunnel->mad);
>> + &ah_attr, wc->smac, vlan_id, &tunnel->mad);
>> }
>>
>> static int mlx4_ib_alloc_pv_bufs(struct mlx4_ib_demux_pv_ctx *ctx, diff --git
>> a/drivers/infiniband/hw/mlx4/mcg.c b/drivers/infiniband/hw/mlx4/mcg.c
>> index ed327e6..86bc158 100644
>> --- a/drivers/infiniband/hw/mlx4/mcg.c
>> +++ b/drivers/infiniband/hw/mlx4/mcg.c
>> @@ -217,7 +217,7 @@ static int send_mad_to_wire(struct mlx4_ib_demux_ctx
>> *ctx, struct ib_mad *mad)
>> spin_unlock(&dev->sm_lock);
>> return mlx4_ib_send_to_wire(dev, mlx4_master_func_num(dev->dev),
>> ctx->port, IB_QPT_GSI, 0, 1, IB_QP1_QKEY,
>> - &ah_attr, NULL, mad);
>> + &ah_attr, NULL, 0xffff, mad);
>> }
>>
>> static int send_mad_to_slave(int slave, struct mlx4_ib_demux_ctx *ctx, diff --
>> git a/drivers/infiniband/hw/mlx4/mlx4_ib.h
>> b/drivers/infiniband/hw/mlx4/mlx4_ib.h
>> index 721540c..42fe035 100644
>> --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
>> +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
>> @@ -761,7 +761,7 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int
>> slave, u8 port, int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8
>> port,
>> enum ib_qp_type dest_qpt, u16 pkey_index, u32
>> remote_qpn,
>> u32 qkey, struct ib_ah_attr *attr, u8 *s_mac,
>> - struct ib_mad *mad);
>> + u16 vlan_id, struct ib_mad *mad);
>>
>> __be64 mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx *ctx);
>>
>> diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
>> index 5889c68..9ab9156 100644
>> --- a/drivers/infiniband/hw/mlx4/qp.c
>> +++ b/drivers/infiniband/hw/mlx4/qp.c
>> @@ -1351,11 +1351,12 @@ static int _mlx4_set_path(struct mlx4_ib_dev
>> *dev, const struct ib_ah_attr *ah, static int mlx4_set_path(struct mlx4_ib_dev
>> *dev, const struct ib_qp_attr *qp,
>> enum ib_qp_attr_mask qp_attr_mask,
>> struct mlx4_ib_qp *mqp,
>> - struct mlx4_qp_path *path, u8 port)
>> + struct mlx4_qp_path *path, u8 port,
>> + u16 vlan_id, u8 *smac)
>> {
>> return _mlx4_set_path(dev, &qp->ah_attr,
>> - mlx4_mac_to_u64((u8 *)qp->smac),
>> - (qp_attr_mask & IB_QP_VID) ? qp->vlan_id : 0xffff,
>> + mlx4_mac_to_u64(smac),
>> + vlan_id,
>> path, &mqp->pri, port);
>> }
>>
>> @@ -1366,9 +1367,8 @@ static int mlx4_set_alt_path(struct mlx4_ib_dev
>> *dev,
>> struct mlx4_qp_path *path, u8 port) {
>> return _mlx4_set_path(dev, &qp->alt_ah_attr,
>> - mlx4_mac_to_u64((u8 *)qp->alt_smac),
>> - (qp_attr_mask & IB_QP_ALT_VID) ?
>> - qp->alt_vlan_id : 0xffff,
>> + 0,
>> + 0xffff,
>> path, &mqp->alt, port);
>> }
>>
>> @@ -1384,7 +1384,8 @@ static void update_mcg_macs(struct mlx4_ib_dev
>> *dev, struct mlx4_ib_qp *qp)
>> }
>> }
>>
>> -static int handle_eth_ud_smac_index(struct mlx4_ib_dev *dev, struct
>> mlx4_ib_qp *qp, u8 *smac,
>> +static int handle_eth_ud_smac_index(struct mlx4_ib_dev *dev,
>> + struct mlx4_ib_qp *qp,
>> struct mlx4_qp_context *context) {
>> u64 u64_mac;
>> @@ -1524,9 +1525,30 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
>> }
>>
>> if (attr_mask & IB_QP_AV) {
>> + u8 port_num = attr_mask & IB_QP_PORT ? attr->port_num :
>> qp->port;
>> + int index = attr->ah_attr.grh.sgid_index;
>> + union ib_gid gid;
>> + struct ib_gid_attr gid_attr;
>> + u16 vlan = 0xffff;
>> + u8 smac[ETH_ALEN];
>> + int status = 0;
>> +
>> + if (rdma_port_get_link_layer(&dev->ib_dev, qp->port) ==
>> + IB_LINK_LAYER_ETHERNET) {
>> + rcu_read_lock();
>> + status = ib_get_cached_gid(ibqp->device, port_num,
>> + index, &gid, &gid_attr);
>> + if (!status) {
>> + vlan = rdma_vlan_dev_vlan_id(gid_attr.ndev);
>> + memcpy(smac, gid_attr.ndev->dev_addr,
>> ETH_ALEN);
>> + }
>> + rcu_read_unlock();
>> + }
>> + if (status)
>> + goto out;
>> +
>> if (mlx4_set_path(dev, attr, attr_mask, qp, &context->pri_path,
>> - attr_mask & IB_QP_PORT ?
>> - attr->port_num : qp->port))
>> + port_num, vlan, smac))
>> goto out;
>>
>> optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH | @@ -
>> 1663,7 +1685,7 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
>> if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_UD ||
>> qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI
>> ||
>> qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI) {
>> - err = handle_eth_ud_smac_index(dev, qp, (u8
>> *)attr->smac, context);
>> + err = handle_eth_ud_smac_index(dev, qp,
>> context);
>> if (err)
>> return -EINVAL;
>> if (qp->mlx4_ib_qp_type ==
>> MLX4_IB_QPT_PROXY_GSI) diff --git a/drivers/infiniband/hw/ocrdma/ocrdma.h
>> b/drivers/infiniband/hw/ocrdma/ocrdma.h
>> index c9780d9..16ee36e 100644
>> --- a/drivers/infiniband/hw/ocrdma/ocrdma.h
>> +++ b/drivers/infiniband/hw/ocrdma/ocrdma.h
>> @@ -36,6 +36,7 @@
>> #include <rdma/ib_verbs.h>
>> #include <rdma/ib_user_verbs.h>
>> #include <rdma/ib_addr.h>
>> +#include <rdma/ib_cache.h>
>>
>> #include <be_roce.h>
>> #include "ocrdma_sli.h"
>> diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
>> b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
>> index d812904..7ecd230 100644
>> --- a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
>> +++ b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
>> @@ -41,10 +41,9 @@
>>
>> static inline int set_av_attr(struct ocrdma_dev *dev, struct ocrdma_ah *ah,
>> struct ib_ah_attr *attr, union ib_gid *sgid,
>> - int pdid, bool *isvlan)
>> + int pdid, bool *isvlan, u16 vlan_tag)
>> {
>> int status = 0;
>> - u16 vlan_tag;
>> struct ocrdma_eth_vlan eth;
>> struct ocrdma_grh grh;
>> int eth_sz;
>> @@ -53,7 +52,6 @@ static inline int set_av_attr(struct ocrdma_dev *dev,
>> struct ocrdma_ah *ah,
>> memset(&grh, 0, sizeof(grh));
>>
>> /* VLAN */
>> - vlan_tag = attr->vlan_id;
>> if (!vlan_tag || (vlan_tag > 0xFFF))
>> vlan_tag = dev->pvid;
>> if (vlan_tag && (vlan_tag < 0x1000)) { @@ -94,9 +92,11 @@ static
>> inline int set_av_attr(struct ocrdma_dev *dev, struct ocrdma_ah *ah, struct
>> ib_ah *ocrdma_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr) {
>> u32 *ahid_addr;
>> - bool isvlan = false;
>> int status;
>> struct ocrdma_ah *ah;
>> + bool isvlan = false;
>> + u16 vlan_tag = 0xffff;
>> + struct ib_gid_attr sgid_attr;
>> struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
>> struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
>> union ib_gid sgid;
>> @@ -114,16 +114,22 @@ struct ib_ah *ocrdma_create_ah(struct ib_pd *ibpd,
>> struct ib_ah_attr *attr)
>> if (status)
>> goto av_err;
>>
>> - status = ocrdma_query_gid(&dev->ibdev, 1, attr->grh.sgid_index,
>> &sgid);
>> + rcu_read_lock();
>> + status = ib_get_cached_gid(&dev->ibdev, 1, attr->grh.sgid_index,
>> &sgid,
>> + &sgid_attr);
>> if (status) {
>> pr_err("%s(): Failed to query sgid, status = %d\n",
>> __func__, status);
>> goto av_conf_err;
>> }
>> + if (sgid_attr.ndev && is_vlan_dev(sgid_attr.ndev))
>> + vlan_tag = vlan_dev_vlan_id(sgid_attr.ndev);
>> + rcu_read_unlock();
>>
>> if (pd->uctx) {
>> status = rdma_addr_find_dmac_by_grh(&sgid, &attr->grh.dgid,
>> - attr->dmac, &attr->vlan_id);
>> + attr->dmac, &vlan_tag,
>> + sgid_attr.ndev->ifindex);
>> if (status) {
>> pr_err("%s(): Failed to resolve dmac from gid."
>> "status = %d\n", __func__, status); @@ -131,7
>> +137,7 @@ struct ib_ah *ocrdma_create_ah(struct ib_pd *ibpd, struct
>> ib_ah_attr *attr)
>> }
>> }
>>
>> - status = set_av_attr(dev, ah, attr, &sgid, pd->id, &isvlan);
>> + status = set_av_attr(dev, ah, attr, &sgid, pd->id, &isvlan, vlan_tag);
>> if (status)
>> goto av_conf_err;
>>
>> diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
>> b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
>> index 31493b1..c0dda74 100644
>> --- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
>> +++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
>> @@ -2428,7 +2428,8 @@ static int ocrdma_set_av_params(struct ocrdma_qp
>> *qp,
>> int status;
>> struct ib_ah_attr *ah_attr = &attrs->ah_attr;
>> union ib_gid sgid, zgid;
>> - u32 vlan_id;
>> + struct ib_gid_attr sgid_attr;
>> + u32 vlan_id = 0xffff;
>> u8 mac_addr[6];
>> struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);
>>
>> @@ -2446,10 +2447,15 @@ static int ocrdma_set_av_params(struct
>> ocrdma_qp *qp,
>> cmd->flags |= OCRDMA_QP_PARA_FLOW_LBL_VALID;
>> memcpy(&cmd->params.dgid[0], &ah_attr->grh.dgid.raw[0],
>> sizeof(cmd->params.dgid));
>> - status = ocrdma_query_gid(&dev->ibdev, 1,
>> - ah_attr->grh.sgid_index, &sgid);
>> - if (status)
>> - return status;
>> +
>> + rcu_read_lock();
>> + status = ib_get_cached_gid(&dev->ibdev, 1, ah_attr->grh.sgid_index,
>> + &sgid, &sgid_attr);
>> + if (!status) {
>> + vlan_id = rdma_vlan_dev_vlan_id(sgid_attr.ndev);
>> + memcpy(mac_addr, sgid_attr.ndev->dev_addr, ETH_ALEN);
>> + }
>> + rcu_read_unlock();
>>
>> memset(&zgid, 0, sizeof(zgid));
>> if (!memcmp(&sgid, &zgid, sizeof(zgid))) @@ -2467,7 +2473,6 @@
>> static int ocrdma_set_av_params(struct ocrdma_qp *qp,
>> ocrdma_cpu_to_le32(&cmd->params.sgid[0], sizeof(cmd-
>>> params.sgid));
>> cmd->params.vlan_dmac_b4_to_b5 = mac_addr[4] | (mac_addr[5] <<
>> 8);
>> if (attr_mask & IB_QP_VID) {
>> - vlan_id = attrs->vlan_id;
>> cmd->params.vlan_dmac_b4_to_b5 |=
>> vlan_id << OCRDMA_QP_PARAMS_VLAN_SHIFT;
>> cmd->flags |= OCRDMA_QP_PARA_VLAN_EN_VALID; diff --git
>> a/include/rdma/ib_addr.h b/include/rdma/ib_addr.h index 3cf32d1..0dfaaa7
>> 100644
>> --- a/include/rdma/ib_addr.h
>> +++ b/include/rdma/ib_addr.h
>> @@ -112,7 +112,7 @@ int rdma_addr_size(struct sockaddr *addr);
>>
>> int rdma_addr_find_smac_by_sgid(union ib_gid *sgid, u8 *smac, u16
>> *vlan_id); int rdma_addr_find_dmac_by_grh(union ib_gid *sgid, union ib_gid
>> *dgid, u8 *smac,
>> - u16 *vlan_id);
>> + u16 *vlan_id, int if_index);
>>
>> static inline u16 ib_addr_get_pkey(struct rdma_dev_addr *dev_addr) { diff --
>> git a/include/rdma/ib_sa.h b/include/rdma/ib_sa.h index 6a1b994..eea01e6
>> 100644
>> --- a/include/rdma/ib_sa.h
>> +++ b/include/rdma/ib_sa.h
>> @@ -154,9 +154,7 @@ struct ib_sa_path_rec {
>> u8 packet_life_time_selector;
>> u8 packet_life_time;
>> u8 preference;
>> - u8 smac[ETH_ALEN];
>> u8 dmac[ETH_ALEN];
>> - u16 vlan_id;
>> int ifindex;
>> struct net *net;
>> };
>> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index
>> 37c3f8f..854e705 100644
>> --- a/include/rdma/ib_verbs.h
>> +++ b/include/rdma/ib_verbs.h
>> @@ -74,6 +74,8 @@ enum ib_gid_type {
>> IB_GID_TYPE_SIZE
>> };
>>
>> +#define ROCE_V2_UDP_DPORT 1021
>> +
>> struct ib_gid_attr {
>> enum ib_gid_type gid_type;
>> struct net_device *ndev;
>> @@ -668,7 +670,6 @@ struct ib_ah_attr {
>> u8 ah_flags;
>> u8 port_num;
>> u8 dmac[ETH_ALEN];
>> - u16 vlan_id;
>> };
>>
>> enum ib_wc_status {
>> @@ -979,10 +980,6 @@ struct ib_qp_attr {
>> u8 rnr_retry;
>> u8 alt_port_num;
>> u8 alt_timeout;
>> - u8 smac[ETH_ALEN];
>> - u8 alt_smac[ETH_ALEN];
>> - u16 vlan_id;
>> - u16 alt_vlan_id;
>> };
>>
>> enum ib_wr_opcode {
>> --
>> 1.7.1
>>
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body
>> of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org More majordomo info at
>> http://vger.kernel.org/majordomo-info.html