[PATCH for-rc 0/6] RDMA/bnxt_re: Misc bug fixes

[PATCH for-rc 0/6] RDMA/bnxt_re: Misc bug fixes

[Selvin Xavier]

This series includes some of the bug fixes for bnxt_re driver.
Please consider this for rc.

Thanks,
Selvin Xavier

Devesh Sharma (2):
  RDMA/bnxt_re: Disable atomic capability on bnxt_re adapters
  RDMA/bnxt_re: Unpin SQ and RQ memory if QP create fails

Selvin Xavier (4):
  RDMA/bnxt_re: Maintain GID index mapping between stack and hardware
  RDMA/bnxt_re: Synchronize destroy_qp with poll_cq
  RDMA/bnxt_re: Fix system crash during load/unload
  RDMA/bnxt_re: Avoid system hang during device un-reg

 drivers/infiniband/hw/bnxt_re/bnxt_re.h  |  5 ++-
 drivers/infiniband/hw/bnxt_re/ib_verbs.c | 75 ++++++++++++++++++++++++--------
 drivers/infiniband/hw/bnxt_re/ib_verbs.h |  2 +
 drivers/infiniband/hw/bnxt_re/main.c     | 25 +++++++++--
 drivers/infiniband/hw/bnxt_re/qplib_fp.c | 21 +++------
 drivers/infiniband/hw/bnxt_re/qplib_fp.h |  4 +-
 drivers/infiniband/hw/bnxt_re/qplib_sp.c | 14 +-----
 7 files changed, 93 insertions(+), 53 deletions(-)

-- 
2.5.5

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH for-rc 1/6] RDMA/bnxt_re: Disable atomic capability on bnxt_re adapters

[Selvin Xavier]

From: Devesh Sharma <devesh.sharma-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>

More testing needs to be done before enabling this feature.
Disabling the feature temporarily

Signed-off-by: Selvin Xavier <selvin.xavier-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>
---
 drivers/infiniband/hw/bnxt_re/ib_verbs.c |  6 ++----
 drivers/infiniband/hw/bnxt_re/qplib_sp.c | 14 +-------------
 2 files changed, 3 insertions(+), 17 deletions(-)

diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 9b8fa77..ce2a6d0 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -174,10 +174,8 @@ int bnxt_re_query_device(struct ib_device *ibdev,
 	ib_attr->max_pd = dev_attr->max_pd;
 	ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
 	ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
-	if (dev_attr->is_atomic) {
-		ib_attr->atomic_cap = IB_ATOMIC_HCA;
-		ib_attr->masked_atomic_cap = IB_ATOMIC_HCA;
-	}
+	ib_attr->atomic_cap = IB_ATOMIC_NONE;
+	ib_attr->masked_atomic_cap = IB_ATOMIC_NONE;
 
 	ib_attr->max_ee_rd_atom = 0;
 	ib_attr->max_res_rd_atom = 0;
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
index c015c18..0305798 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
@@ -52,18 +52,6 @@ const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0,
 
 /* Device */
 
-static bool bnxt_qplib_is_atomic_cap(struct bnxt_qplib_rcfw *rcfw)
-{
-	int rc;
-	u16 pcie_ctl2;
-
-	rc = pcie_capability_read_word(rcfw->pdev, PCI_EXP_DEVCTL2,
-				       &pcie_ctl2);
-	if (rc)
-		return false;
-	return !!(pcie_ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ);
-}
-
 static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw,
 				     char *fw_ver)
 {
@@ -165,7 +153,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
 		attr->tqm_alloc_reqs[i * 4 + 3] = *(++tqm_alloc);
 	}
 
-	attr->is_atomic = bnxt_qplib_is_atomic_cap(rcfw);
+	attr->is_atomic = 0;
 bail:
 	bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
 	return rc;
-- 
2.5.5

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH for-rc 2/6] RDMA/bnxt_re: Maintain GID index mapping between stack and hardware

[Selvin Xavier]

Every GID has a duplicate entry in the stack for RoCE v2.
HW table maintain only single entries. Currently the hw index
is calculated by dividing the stack index by two. This is prone to
failure after a series of add/delete gid operation in random order.

Maintain a mapping between stack GID index and hardware
index to avoid failure.

Signed-off-by: Selvin Xavier <selvin.xavier-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>
---
 drivers/infiniband/hw/bnxt_re/bnxt_re.h  |  3 +++
 drivers/infiniband/hw/bnxt_re/ib_verbs.c | 21 ++++++++++-----------
 drivers/infiniband/hw/bnxt_re/main.c     | 13 +++++++++++++
 3 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
index ca32057..5692335 100644
--- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
+++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
@@ -109,6 +109,7 @@ struct bnxt_re_sqp_entries {
 #define BNXT_RE_MAX_MSIX		9
 #define BNXT_RE_AEQ_IDX			0
 #define BNXT_RE_NQ_IDX			1
+#define BNXT_RE_MAX_SGID_ENTRIES	256
 
 struct bnxt_re_dev {
 	struct ib_device		ibdev;
@@ -170,6 +171,8 @@ struct bnxt_re_dev {
 	u32 is_virtfn;
 	u32 num_vfs;
 	struct bnxt_qplib_roce_stats	stats;
+	/* Array to handle gid mapping */
+	char				*gid_map;
 };
 
 #define to_bnxt_re_dev(ptr, member)	\
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index ce2a6d0..7943707 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -348,6 +348,7 @@ int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
 			return -EFAULT;
 		}
 		ctx->refcnt--;
+		rdev->gid_map[index] = -1;
 		if (!ctx->refcnt) {
 			rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, true);
 			if (rc) {
@@ -386,6 +387,8 @@ int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
 		ctx_tbl = sgid_tbl->ctx;
 		ctx_tbl[tbl_idx]->refcnt++;
 		*context = ctx_tbl[tbl_idx];
+		/* tbl_idx is the HW table index and index is the stack index */
+		rdev->gid_map[index] = tbl_idx;
 		return 0;
 	}
 
@@ -401,6 +404,8 @@ int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
 	ctx->idx = tbl_idx;
 	ctx->refcnt = 1;
 	ctx_tbl[tbl_idx] = ctx;
+	/* tbl_idx is the HW table index and index is the stack index */
+	rdev->gid_map[index] = tbl_idx;
 	*context = ctx;
 
 	return rc;
@@ -691,12 +696,8 @@ struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd,
 	/* Supply the configuration for the HW */
 	memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw,
 	       sizeof(union ib_gid));
-	/*
-	 * If RoCE V2 is enabled, stack will have two entries for
-	 * each GID entry. Avoiding this duplicte entry in HW. Dividing
-	 * the GID index by 2 for RoCE V2
-	 */
-	ah->qplib_ah.sgid_index = grh->sgid_index / 2;
+	/* Retrieve the HW index from the driver SGID map */
+	ah->qplib_ah.sgid_index = rdev->gid_map[ah_attr->grh.sgid_index];
 	ah->qplib_ah.host_sgid_index = grh->sgid_index;
 	ah->qplib_ah.traffic_class = grh->traffic_class;
 	ah->qplib_ah.flow_label = grh->flow_label;
@@ -1641,11 +1642,9 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 		memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw,
 		       sizeof(qp->qplib_qp.ah.dgid.data));
 		qp->qplib_qp.ah.flow_label = grh->flow_label;
-		/* If RoCE V2 is enabled, stack will have two entries for
-		 * each GID entry. Avoiding this duplicte entry in HW. Dividing
-		 * the GID index by 2 for RoCE V2
-		 */
-		qp->qplib_qp.ah.sgid_index = grh->sgid_index / 2;
+		/* Retrieve the HW index from the driver SGID map */
+		qp->qplib_qp.ah.sgid_index =
+			rdev->gid_map[qp_attr->ah_attr.grh.sgid_index];
 		qp->qplib_qp.ah.host_sgid_index = grh->sgid_index;
 		qp->qplib_qp.ah.hop_limit = grh->hop_limit;
 		qp->qplib_qp.ah.traffic_class = grh->traffic_class;
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index 508d00a..3d5bbf9 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -658,6 +658,7 @@ static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev)
 	synchronize_rcu();
 	flush_workqueue(bnxt_re_wq);
 
+	kfree(rdev->gid_map);
 	ib_dealloc_device(&rdev->ibdev);
 	/* rdev is gone */
 }
@@ -666,6 +667,7 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
 					   struct bnxt_en_dev *en_dev)
 {
 	struct bnxt_re_dev *rdev;
+	u32 count;
 
 	/* Allocate bnxt_re_dev instance here */
 	rdev = (struct bnxt_re_dev *)ib_alloc_device(sizeof(*rdev));
@@ -689,6 +691,17 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
 	rdev->cosq[0] = 0xFFFF;
 	rdev->cosq[1] = 0xFFFF;
 
+	rdev->gid_map = kzalloc(sizeof(*rdev->gid_map) *
+				BNXT_RE_MAX_SGID_ENTRIES,
+				GFP_KERNEL);
+	if (!rdev->gid_map) {
+		ib_dealloc_device(&rdev->ibdev);
+		return NULL;
+	}
+
+	for (count = 0; count < BNXT_RE_MAX_SGID_ENTRIES; count++)
+		rdev->gid_map[count] = -1;
+
 	mutex_lock(&bnxt_re_dev_lock);
 	list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list);
 	mutex_unlock(&bnxt_re_dev_lock);
-- 
2.5.5

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH for-rc 2/6] RDMA/bnxt_re: Maintain GID index mapping between stack and hardware

[Jason Gunthorpe]

On Thu, Feb 15, 2018 at 09:20:09PM -0800, Selvin Xavier wrote:
> Every GID has a duplicate entry in the stack for RoCE v2.
> HW table maintain only single entries. Currently the hw index
> is calculated by dividing the stack index by two. This is prone to
> failure after a series of add/delete gid operation in random order.
> 
> Maintain a mapping between stack GID index and hardware
> index to avoid failure.
> 
> Signed-off-by: Selvin Xavier <selvin.xavier-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>
>  drivers/infiniband/hw/bnxt_re/bnxt_re.h  |  3 +++
>  drivers/infiniband/hw/bnxt_re/ib_verbs.c | 21 ++++++++++-----------
>  drivers/infiniband/hw/bnxt_re/main.c     | 13 +++++++++++++
>  3 files changed, 26 insertions(+), 11 deletions(-)
> 
> diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
> index ca32057..5692335 100644
> +++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
> @@ -109,6 +109,7 @@ struct bnxt_re_sqp_entries {
>  #define BNXT_RE_MAX_MSIX		9
>  #define BNXT_RE_AEQ_IDX			0
>  #define BNXT_RE_NQ_IDX			1
> +#define BNXT_RE_MAX_SGID_ENTRIES	256
>  
>  struct bnxt_re_dev {
>  	struct ib_device		ibdev;
> @@ -170,6 +171,8 @@ struct bnxt_re_dev {
>  	u32 is_virtfn;
>  	u32 num_vfs;
>  	struct bnxt_qplib_roce_stats	stats;
> +	/* Array to handle gid mapping */
> +	char				*gid_map;

char would need to be a s8

>  #define to_bnxt_re_dev(ptr, member)	\
> diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
> index ce2a6d0..7943707 100644
> +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
> @@ -348,6 +348,7 @@ int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
>  			return -EFAULT;
>  		}
>  		ctx->refcnt--;
> +		rdev->gid_map[index] = -1;
>  		if (!ctx->refcnt) {
>  			rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, true);
>  			if (rc) {
> @@ -386,6 +387,8 @@ int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
>  		ctx_tbl = sgid_tbl->ctx;
>  		ctx_tbl[tbl_idx]->refcnt++;
>  		*context = ctx_tbl[tbl_idx];
> +		/* tbl_idx is the HW table index and index is the stack index */
> +		rdev->gid_map[index] = tbl_idx;

It would be a lot better to store tbl_idx in the driver's context, that
is what context is for.

> -	/*
> -	 * If RoCE V2 is enabled, stack will have two entries for
> -	 * each GID entry. Avoiding this duplicte entry in HW. Dividing
> -	 * the GID index by 2 for RoCE V2
> -	 */
> -	ah->qplib_ah.sgid_index = grh->sgid_index / 2;
> +	/* Retrieve the HW index from the driver SGID map */
> +	ah->qplib_ah.sgid_index = rdev->gid_map[ah_attr->grh.sgid_index];

You'd need a new api to return the driver context for a gid index, but
that shouldn't be a problem.

Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH for-rc 2/6] RDMA/bnxt_re: Maintain GID index mapping between stack and hardware

[Selvin Xavier]

On Fri, Feb 16, 2018 at 9:37 PM, Jason Gunthorpe <jgg-uk2M96/98Pc@public.gmane.org> wrote:
> On Thu, Feb 15, 2018 at 09:20:09PM -0800, Selvin Xavier wrote:
>> Every GID has a duplicate entry in the stack for RoCE v2.
>> HW table maintain only single entries. Currently the hw index
>> is calculated by dividing the stack index by two. This is prone to
>> failure after a series of add/delete gid operation in random order.
>>
>> Maintain a mapping between stack GID index and hardware
>> index to avoid failure.
>>
>> Signed-off-by: Selvin Xavier <selvin.xavier-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>
>>  drivers/infiniband/hw/bnxt_re/bnxt_re.h  |  3 +++
>>  drivers/infiniband/hw/bnxt_re/ib_verbs.c | 21 ++++++++++-----------
>>  drivers/infiniband/hw/bnxt_re/main.c     | 13 +++++++++++++
>>  3 files changed, 26 insertions(+), 11 deletions(-)
>>
>> diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
>> index ca32057..5692335 100644
>> +++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
>> @@ -109,6 +109,7 @@ struct bnxt_re_sqp_entries {
>>  #define BNXT_RE_MAX_MSIX             9
>>  #define BNXT_RE_AEQ_IDX                      0
>>  #define BNXT_RE_NQ_IDX                       1
>> +#define BNXT_RE_MAX_SGID_ENTRIES     256
>>
>>  struct bnxt_re_dev {
>>       struct ib_device                ibdev;
>> @@ -170,6 +171,8 @@ struct bnxt_re_dev {
>>       u32 is_virtfn;
>>       u32 num_vfs;
>>       struct bnxt_qplib_roce_stats    stats;
>> +     /* Array to handle gid mapping */
>> +     char                            *gid_map;
>
> char would need to be a s8
Sure.. i will make the change
>
>>  #define to_bnxt_re_dev(ptr, member)  \
>> diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
>> index ce2a6d0..7943707 100644
>> +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
>> @@ -348,6 +348,7 @@ int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
>>                       return -EFAULT;
>>               }
>>               ctx->refcnt--;
>> +             rdev->gid_map[index] = -1;
>>               if (!ctx->refcnt) {
>>                       rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, true);
>>                       if (rc) {
>> @@ -386,6 +387,8 @@ int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
>>               ctx_tbl = sgid_tbl->ctx;
>>               ctx_tbl[tbl_idx]->refcnt++;
>>               *context = ctx_tbl[tbl_idx];
>> +             /* tbl_idx is the HW table index and index is the stack index */
>> +             rdev->gid_map[index] = tbl_idx;
>
> It would be a lot better to store tbl_idx in the driver's context, that
> is what context is for.
We do hold this currently in the driver context. The mapping was to retrieve the
gid_index to hw_index, which needs an API otherwise as you mentioned below.
>
>> -     /*
>> -      * If RoCE V2 is enabled, stack will have two entries for
>> -      * each GID entry. Avoiding this duplicte entry in HW. Dividing
>> -      * the GID index by 2 for RoCE V2
>> -      */
>> -     ah->qplib_ah.sgid_index = grh->sgid_index / 2;
>> +     /* Retrieve the HW index from the driver SGID map */
>> +     ah->qplib_ah.sgid_index = rdev->gid_map[ah_attr->grh.sgid_index];
>
> You'd need a new api to return the driver context for a gid index, but
> that shouldn't be a problem.
I will post a stack change and rework on this patch.
Shall i drop this patch from this series and re-post  remaining?
>
> Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH for-rc 3/6] RDMA/bnxt_re: Unpin SQ and RQ memory if QP create fails

[Selvin Xavier]

From: Devesh Sharma <devesh.sharma-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>

Driver leaves the QP memory pinned if QP create command
fails from the FW. Avoids this scenario by adding a proper
exit path if the FW command fails.

Signed-off-by: Devesh Sharma <devesh.sharma-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>
Signed-off-by: Selvin Xavier <selvin.xavier-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>
---
 drivers/infiniband/hw/bnxt_re/ib_verbs.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 7943707..b7eb104 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -1184,7 +1184,7 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
 		rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
 		if (rc) {
 			dev_err(rdev_to_dev(rdev), "Failed to create HW QP");
-			goto fail;
+			goto free_umem;
 		}
 	}
 
@@ -1212,6 +1212,13 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
 	return &qp->ib_qp;
 qp_destroy:
 	bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
+free_umem:
+	if (udata) {
+		if (qp->rumem)
+			ib_umem_release(qp->rumem);
+		if (qp->sumem)
+			ib_umem_release(qp->sumem);
+	}
 fail:
 	kfree(qp);
 	return ERR_PTR(rc);
-- 
2.5.5

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH for-rc 4/6] RDMA/bnxt_re: Synchronize destroy_qp with poll_cq

[Selvin Xavier]

Avoid system crash when destroy_qp is invoked while
the driver is processing the poll_cq. Synchronize these
functions using the cq_lock.

Signed-off-by: Selvin Xavier <selvin.xavier-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>
---
 drivers/infiniband/hw/bnxt_re/ib_verbs.c | 39 +++++++++++++++++++++++++++++---
 drivers/infiniband/hw/bnxt_re/ib_verbs.h |  2 ++
 drivers/infiniband/hw/bnxt_re/qplib_fp.c | 21 +++++------------
 drivers/infiniband/hw/bnxt_re/qplib_fp.h |  4 +++-
 4 files changed, 47 insertions(+), 19 deletions(-)

diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index b7eb104..9cb9928 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -786,20 +786,51 @@ int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
 	return 0;
 }
 
+static unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
+	__acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&qp->scq->cq_lock, flags);
+	if (qp->rcq != qp->scq)
+		spin_lock(&qp->rcq->cq_lock);
+	else
+		__acquire(&qp->rcq->cq_lock);
+
+	return flags;
+}
+
+static void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
+			       unsigned long flags)
+	__releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
+{
+	if (qp->rcq != qp->scq)
+		spin_unlock(&qp->rcq->cq_lock);
+	else
+		__release(&qp->rcq->cq_lock);
+	spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
+}
+
 /* Queue Pairs */
 int bnxt_re_destroy_qp(struct ib_qp *ib_qp)
 {
 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
 	struct bnxt_re_dev *rdev = qp->rdev;
 	int rc;
+	unsigned int flags;
 
 	bnxt_qplib_flush_cqn_wq(&qp->qplib_qp);
-	bnxt_qplib_del_flush_qp(&qp->qplib_qp);
 	rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
 	if (rc) {
 		dev_err(rdev_to_dev(rdev), "Failed to destroy HW QP");
 		return rc;
 	}
+
+	flags = bnxt_re_lock_cqs(qp);
+	bnxt_qplib_clean_qp(&qp->qplib_qp);
+	bnxt_re_unlock_cqs(qp, flags);
+	bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
+
 	if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) {
 		rc = bnxt_qplib_destroy_ah(&rdev->qplib_res,
 					   &rdev->sqp_ah->qplib_ah);
@@ -809,7 +840,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp)
 			return rc;
 		}
 
-		bnxt_qplib_del_flush_qp(&qp->qplib_qp);
+		bnxt_qplib_clean_qp(&qp->qplib_qp);
 		rc = bnxt_qplib_destroy_qp(&rdev->qplib_res,
 					   &rdev->qp1_sqp->qplib_qp);
 		if (rc) {
@@ -1068,6 +1099,7 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
 			goto fail;
 		}
 		qp->qplib_qp.scq = &cq->qplib_cq;
+		qp->scq = cq;
 	}
 
 	if (qp_init_attr->recv_cq) {
@@ -1079,6 +1111,7 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
 			goto fail;
 		}
 		qp->qplib_qp.rcq = &cq->qplib_cq;
+		qp->rcq = cq;
 	}
 
 	if (qp_init_attr->srq) {
@@ -1612,7 +1645,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 			dev_dbg(rdev_to_dev(rdev),
 				"Move QP = %p out of flush list\n",
 				qp);
-			bnxt_qplib_del_flush_qp(&qp->qplib_qp);
+			bnxt_qplib_clean_qp(&qp->qplib_qp);
 		}
 	}
 	if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
index 423ebe0..b88a48d 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
@@ -89,6 +89,8 @@ struct bnxt_re_qp {
 	/* QP1 */
 	u32			send_psn;
 	struct ib_ud_header	qp1_hdr;
+	struct bnxt_re_cq	*scq;
+	struct bnxt_re_cq	*rcq;
 };
 
 struct bnxt_re_cq {
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 8b5f11a..52df919 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -173,7 +173,7 @@ static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
 	}
 }
 
-void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
+void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp)
 {
 	unsigned long flags;
 
@@ -1417,7 +1417,6 @@ int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 	struct cmdq_destroy_qp req;
 	struct creq_destroy_qp_resp resp;
-	unsigned long flags;
 	u16 cmd_flags = 0;
 	int rc;
 
@@ -1435,19 +1434,12 @@ int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
 		return rc;
 	}
 
-	/* Must walk the associated CQs to nullified the QP ptr */
-	spin_lock_irqsave(&qp->scq->hwq.lock, flags);
-
-	__clean_cq(qp->scq, (u64)(unsigned long)qp);
-
-	if (qp->rcq && qp->rcq != qp->scq) {
-		spin_lock(&qp->rcq->hwq.lock);
-		__clean_cq(qp->rcq, (u64)(unsigned long)qp);
-		spin_unlock(&qp->rcq->hwq.lock);
-	}
-
-	spin_unlock_irqrestore(&qp->scq->hwq.lock, flags);
+	return 0;
+}
 
+void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res,
+			    struct bnxt_qplib_qp *qp)
+{
 	bnxt_qplib_free_qp_hdr_buf(res, qp);
 	bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);
 	kfree(qp->sq.swq);
@@ -1460,7 +1452,6 @@ int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
 	if (qp->orrq.max_elements)
 		bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
 
-	return 0;
 }
 
 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
index 211b27a..ca0a2ff 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
@@ -478,6 +478,9 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
+void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp);
+void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res,
+			    struct bnxt_qplib_qp *qp);
 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
 				struct bnxt_qplib_sge *sge);
 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
@@ -500,7 +503,6 @@ void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type);
 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq);
 int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq);
 void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp);
-void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp);
 void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp,
 				 unsigned long *flags);
 void bnxt_qplib_release_cq_locks(struct bnxt_qplib_qp *qp,
-- 
2.5.5

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH for-rc 4/6] RDMA/bnxt_re: Synchronize destroy_qp with poll_cq

[Leon Romanovsky]

[-- Attachment #1: Type: text/plain, Size: 1786 bytes --]

On Thu, Feb 15, 2018 at 09:20:11PM -0800, Selvin Xavier wrote:
> Avoid system crash when destroy_qp is invoked while
> the driver is processing the poll_cq. Synchronize these
> functions using the cq_lock.
>
> Signed-off-by: Selvin Xavier <selvin.xavier-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>
> ---
>  drivers/infiniband/hw/bnxt_re/ib_verbs.c | 39 +++++++++++++++++++++++++++++---
>  drivers/infiniband/hw/bnxt_re/ib_verbs.h |  2 ++
>  drivers/infiniband/hw/bnxt_re/qplib_fp.c | 21 +++++------------
>  drivers/infiniband/hw/bnxt_re/qplib_fp.h |  4 +++-
>  4 files changed, 47 insertions(+), 19 deletions(-)
>
> diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
> index b7eb104..9cb9928 100644
> --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
> +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
> @@ -786,20 +786,51 @@ int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
>  	return 0;
>  }
>
> +static unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
> +	__acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
> +{
> +	unsigned long flags;
> +
> +	spin_lock_irqsave(&qp->scq->cq_lock, flags);
> +	if (qp->rcq != qp->scq)
> +		spin_lock(&qp->rcq->cq_lock);
> +	else
> +		__acquire(&qp->rcq->cq_lock);

Sorry for naive question, why do you need __acquire/__release? And why
can't spin_is_locked(&qp->rcq->cq_lock) be used for unlock code?


> +
> +	return flags;
> +}
> +
> +static void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
> +			       unsigned long flags)
> +	__releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
> +{
> +	if (qp->rcq != qp->scq)
> +		spin_unlock(&qp->rcq->cq_lock);
> +	else
> +		__release(&qp->rcq->cq_lock);
> +	spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
> +}
> +

Thanks

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH for-rc 4/6] RDMA/bnxt_re: Synchronize destroy_qp with poll_cq

[Selvin Xavier]

On Fri, Feb 16, 2018 at 1:08 PM, Leon Romanovsky <leon-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org> wrote:
> On Thu, Feb 15, 2018 at 09:20:11PM -0800, Selvin Xavier wrote:
>> Avoid system crash when destroy_qp is invoked while
>> the driver is processing the poll_cq. Synchronize these
>> functions using the cq_lock.
>>
>> Signed-off-by: Selvin Xavier <selvin.xavier-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>
>> ---
>>  drivers/infiniband/hw/bnxt_re/ib_verbs.c | 39 +++++++++++++++++++++++++++++---
>>  drivers/infiniband/hw/bnxt_re/ib_verbs.h |  2 ++
>>  drivers/infiniband/hw/bnxt_re/qplib_fp.c | 21 +++++------------
>>  drivers/infiniband/hw/bnxt_re/qplib_fp.h |  4 +++-
>>  4 files changed, 47 insertions(+), 19 deletions(-)
>>
>> diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
>> index b7eb104..9cb9928 100644
>> --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
>> +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
>> @@ -786,20 +786,51 @@ int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
>>       return 0;
>>  }
>>
>> +static unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
>> +     __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
>> +{
>> +     unsigned long flags;
>> +
>> +     spin_lock_irqsave(&qp->scq->cq_lock, flags);
>> +     if (qp->rcq != qp->scq)
>> +             spin_lock(&qp->rcq->cq_lock);
>> +     else
>> +             __acquire(&qp->rcq->cq_lock);
>
> Sorry for naive question, why do you need __acquire/__release? And why
> can't spin_is_locked(&qp->rcq->cq_lock) be used for unlock code?
>
I used __acquire and __release for satisfying the sparse. Sparse used to
give warnings  "context  imbalance - wrong count at exit" since the locking
and unlocking happens from two different functions. seen similar implementation
in other drivers too.

I feel spin_is_locked also can be used.. but it doesn't avoid the RCQ
!= SCQ check.
Also, if feel the code symmetry is better in this way.
>
>> +
>> +     return flags;
>> +}
>> +
>> +static void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
>> +                            unsigned long flags)
>> +     __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
>> +{
>> +     if (qp->rcq != qp->scq)
>> +             spin_unlock(&qp->rcq->cq_lock);
>> +     else
>> +             __release(&qp->rcq->cq_lock);
>> +     spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
>> +}
>> +
>
> Thanks
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH for-rc 4/6] RDMA/bnxt_re: Synchronize destroy_qp with poll_cq

[Leon Romanovsky]

[-- Attachment #1: Type: text/plain, Size: 2896 bytes --]

On Fri, Feb 16, 2018 at 05:09:15PM +0530, Selvin Xavier wrote:
> On Fri, Feb 16, 2018 at 1:08 PM, Leon Romanovsky <leon-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org> wrote:
> > On Thu, Feb 15, 2018 at 09:20:11PM -0800, Selvin Xavier wrote:
> >> Avoid system crash when destroy_qp is invoked while
> >> the driver is processing the poll_cq. Synchronize these
> >> functions using the cq_lock.
> >>
> >> Signed-off-by: Selvin Xavier <selvin.xavier-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>
> >> ---
> >>  drivers/infiniband/hw/bnxt_re/ib_verbs.c | 39 +++++++++++++++++++++++++++++---
> >>  drivers/infiniband/hw/bnxt_re/ib_verbs.h |  2 ++
> >>  drivers/infiniband/hw/bnxt_re/qplib_fp.c | 21 +++++------------
> >>  drivers/infiniband/hw/bnxt_re/qplib_fp.h |  4 +++-
> >>  4 files changed, 47 insertions(+), 19 deletions(-)
> >>
> >> diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
> >> index b7eb104..9cb9928 100644
> >> --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
> >> +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
> >> @@ -786,20 +786,51 @@ int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
> >>       return 0;
> >>  }
> >>
> >> +static unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
> >> +     __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
> >> +{
> >> +     unsigned long flags;
> >> +
> >> +     spin_lock_irqsave(&qp->scq->cq_lock, flags);
> >> +     if (qp->rcq != qp->scq)
> >> +             spin_lock(&qp->rcq->cq_lock);
> >> +     else
> >> +             __acquire(&qp->rcq->cq_lock);
> >
> > Sorry for naive question, why do you need __acquire/__release? And why
> > can't spin_is_locked(&qp->rcq->cq_lock) be used for unlock code?
> >
> I used __acquire and __release for satisfying the sparse. Sparse used to
> give warnings  "context  imbalance - wrong count at exit" since the locking
> and unlocking happens from two different functions. seen similar implementation
> in other drivers too.

I see, actually I think that sparse complain was right and you went to
far by wrapping standard calls. I see that it is very popular in bnxt_re
code and the problem with that, that it makes very hard to refactor such
code in case of some global change/improvement.

>
> I feel spin_is_locked also can be used.. but it doesn't avoid the RCQ
> != SCQ check.
> Also, if feel the code symmetry is better in this way.
> >
> >> +
> >> +     return flags;
> >> +}
> >> +
> >> +static void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
> >> +                            unsigned long flags)
> >> +     __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
> >> +{
> >> +     if (qp->rcq != qp->scq)
> >> +             spin_unlock(&qp->rcq->cq_lock);
> >> +     else
> >> +             __release(&qp->rcq->cq_lock);
> >> +     spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
> >> +}
> >> +
> >
> > Thanks

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

[PATCH for-rc 5/6] RDMA/bnxt_re: Fix system crash during load/unload

[Selvin Xavier]

During driver unload, the driver proceeds with cleanup
without waiting for the scheduled events. So the device
pointers get freed up and driver crashes when the events
are scheduled later.

Flush the bnxt_re_task work queue before starting
device removal.

Signed-off-by: Selvin Xavier <selvin.xavier-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>
---
 drivers/infiniband/hw/bnxt_re/main.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index 3d5bbf9..560520c 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -1591,6 +1591,11 @@ static void __exit bnxt_re_mod_exit(void)
 	*/
 	list_for_each_entry_safe_reverse(rdev, next, &to_be_deleted, list) {
 		dev_info(rdev_to_dev(rdev), "Unregistering Device");
+		/*
+		 * Flush out any scheduled tasks before destroying the
+		 * resources
+		 */
+		flush_workqueue(bnxt_re_wq);
 		bnxt_re_dev_stop(rdev);
 		bnxt_re_ib_unreg(rdev, true);
 		bnxt_re_remove_one(rdev);
-- 
2.5.5

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH for-rc 6/6] RDMA/bnxt_re: Avoid system hang during device un-reg

[Selvin Xavier]

BNXT_RE_FLAG_TASK_IN_PROG doesn't handle multiple work
requests posted together. Track schedule of multiple
workqueue items by maintaining a per device counter
and proceed with IB dereg only if this counter is zero.
flush_workqueue is no longer required from
NETDEV_UNREGISTER path.

Signed-off-by: Selvin Xavier <selvin.xavier-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>
---
 drivers/infiniband/hw/bnxt_re/bnxt_re.h | 2 +-
 drivers/infiniband/hw/bnxt_re/main.c    | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
index 5692335..7463067 100644
--- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
+++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
@@ -121,7 +121,6 @@ struct bnxt_re_dev {
 #define BNXT_RE_FLAG_HAVE_L2_REF		3
 #define BNXT_RE_FLAG_RCFW_CHANNEL_EN		4
 #define BNXT_RE_FLAG_QOS_WORK_REG		5
-#define BNXT_RE_FLAG_TASK_IN_PROG		6
 #define BNXT_RE_FLAG_ISSUE_ROCE_STATS          29
 	struct net_device		*netdev;
 	unsigned int			version, major, minor;
@@ -159,6 +158,7 @@ struct bnxt_re_dev {
 	atomic_t			srq_count;
 	atomic_t			mr_count;
 	atomic_t			mw_count;
+	atomic_t			sched_count;
 	/* Max of 2 lossless traffic class supported per port */
 	u16				cosq[2];
 
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index 560520c..b1aa5a7 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -656,7 +656,6 @@ static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev)
 	mutex_unlock(&bnxt_re_dev_lock);
 
 	synchronize_rcu();
-	flush_workqueue(bnxt_re_wq);
 
 	kfree(rdev->gid_map);
 	ib_dealloc_device(&rdev->ibdev);
@@ -1454,7 +1453,7 @@ static void bnxt_re_task(struct work_struct *work)
 		break;
 	}
 	smp_mb__before_atomic();
-	clear_bit(BNXT_RE_FLAG_TASK_IN_PROG, &rdev->flags);
+	atomic_dec(&rdev->sched_count);
 	kfree(re_work);
 }
 
@@ -1516,7 +1515,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier,
 		/* netdev notifier will call NETDEV_UNREGISTER again later since
 		 * we are still holding the reference to the netdev
 		 */
-		if (test_bit(BNXT_RE_FLAG_TASK_IN_PROG, &rdev->flags))
+		if (atomic_read(&rdev->sched_count) > 0)
 			goto exit;
 		bnxt_re_ib_unreg(rdev, false);
 		bnxt_re_remove_one(rdev);
@@ -1536,7 +1535,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier,
 			re_work->vlan_dev = (real_dev == netdev ?
 					     NULL : netdev);
 			INIT_WORK(&re_work->work, bnxt_re_task);
-			set_bit(BNXT_RE_FLAG_TASK_IN_PROG, &rdev->flags);
+			atomic_inc(&rdev->sched_count);
 			queue_work(bnxt_re_wq, &re_work->work);
 		}
 	}
-- 
2.5.5

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html