[PATCH 0/5] cnic: Bug fixes.

[PATCH 0/5] cnic: Bug fixes.

[Michael Chan]

This patchset fixes a panic on ia64 and some locking issues.
Please review.  Thanks.

[PATCH 1/5] cnic: Fix symbol_put_addr() panic on ia64.

[Michael Chan]

When the cnic driver tries to grab a symbol from bnx2 when bnx2 is
running init code, symbol_get() will succeed but symbol_put_addr()
will hit BUG() a moment later.  module_text_address() fails because
bnx2 is still in init code.

This is fixed by using symbol_put() instead which does the exact
opposite of symbol_get().

Signed-off-by: Michael Chan <mchan@broadcom.com>
---
 drivers/net/cnic.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/drivers/net/cnic.c b/drivers/net/cnic.c
index 4869d77..ecde186 100644
--- a/drivers/net/cnic.c
+++ b/drivers/net/cnic.c
@@ -2543,7 +2543,7 @@ static struct cnic_dev *init_bnx2_cnic(struct net_device *dev)
 	probe = symbol_get(bnx2_cnic_probe);
 	if (probe) {
 		ethdev = (*probe)(dev);
-		symbol_put_addr(probe);
+		symbol_put(bnx2_cnic_probe);
 	}
 	if (!ethdev)
 		return NULL;
-- 
1.5.6.GIT

[PATCH 2/5] cnic: Refine registration with bnx2.

[Michael Chan]

Register and unregister with bnx2 during NETDEV_UP and NETDEV_DOWN
events.  This simplifies the sequence of events and allows locking
fixes in the next patch.

Signed-off-by: Michael Chan <mchan@broadcom.com>
Reviewed-by: Benjamin Li <benli@broadcom.com>
---
 drivers/net/cnic.c |   49 ++++++++++++++++++++++++++++++++++++-------------
 1 files changed, 36 insertions(+), 13 deletions(-)

diff --git a/drivers/net/cnic.c b/drivers/net/cnic.c
index ecde186..2db81f0 100644
--- a/drivers/net/cnic.c
+++ b/drivers/net/cnic.c
@@ -2393,21 +2393,45 @@ static int cnic_start_bnx2_hw(struct cnic_dev *dev)
 	return 0;
 }
 
-static int cnic_start_hw(struct cnic_dev *dev)
+static int cnic_register_netdev(struct cnic_dev *dev)
 {
 	struct cnic_local *cp = dev->cnic_priv;
 	struct cnic_eth_dev *ethdev = cp->ethdev;
 	int err;
 
-	if (test_bit(CNIC_F_CNIC_UP, &dev->flags))
-		return -EALREADY;
+	if (!ethdev)
+		return -ENODEV;
+
+	if (ethdev->drv_state & CNIC_DRV_STATE_REGD)
+		return 0;
 
 	err = ethdev->drv_register_cnic(dev->netdev, cp->cnic_ops, dev);
-	if (err) {
+	if (err)
 		printk(KERN_ERR PFX "%s: register_cnic failed\n",
 		       dev->netdev->name);
-		goto err2;
-	}
+
+	return err;
+}
+
+static void cnic_unregister_netdev(struct cnic_dev *dev)
+{
+	struct cnic_local *cp = dev->cnic_priv;
+	struct cnic_eth_dev *ethdev = cp->ethdev;
+
+	if (!ethdev)
+		return;
+
+	ethdev->drv_unregister_cnic(dev->netdev);
+}
+
+static int cnic_start_hw(struct cnic_dev *dev)
+{
+	struct cnic_local *cp = dev->cnic_priv;
+	struct cnic_eth_dev *ethdev = cp->ethdev;
+	int err;
+
+	if (test_bit(CNIC_F_CNIC_UP, &dev->flags))
+		return -EALREADY;
 
 	dev->regview = ethdev->io_base;
 	cp->chip_id = ethdev->chip_id;
@@ -2438,18 +2462,13 @@ static int cnic_start_hw(struct cnic_dev *dev)
 	return 0;
 
 err1:
-	ethdev->drv_unregister_cnic(dev->netdev);
 	cp->free_resc(dev);
 	pci_dev_put(dev->pcidev);
-err2:
 	return err;
 }
 
 static void cnic_stop_bnx2_hw(struct cnic_dev *dev)
 {
-	struct cnic_local *cp = dev->cnic_priv;
-	struct cnic_eth_dev *ethdev = cp->ethdev;
-
 	cnic_disable_bnx2_int_sync(dev);
 
 	cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 0);
@@ -2461,8 +2480,6 @@ static void cnic_stop_bnx2_hw(struct cnic_dev *dev)
 	cnic_setup_5709_context(dev, 0);
 	cnic_free_irq(dev);
 
-	ethdev->drv_unregister_cnic(dev->netdev);
-
 	cnic_free_resc(dev);
 }
 
@@ -2646,6 +2663,10 @@ static int cnic_netdev_event(struct notifier_block *this, unsigned long event,
 		else if (event == NETDEV_UNREGISTER)
 			cnic_ulp_exit(dev);
 		else if (event == NETDEV_UP) {
+			if (cnic_register_netdev(dev) != 0) {
+				cnic_put(dev);
+				goto done;
+			}
 			mutex_lock(&cnic_lock);
 			if (!cnic_start_hw(dev))
 				cnic_ulp_start(dev);
@@ -2672,6 +2693,7 @@ static int cnic_netdev_event(struct notifier_block *this, unsigned long event,
 			cnic_ulp_stop(dev);
 			cnic_stop_hw(dev);
 			mutex_unlock(&cnic_lock);
+			cnic_unregister_netdev(dev);
 		} else if (event == NETDEV_UNREGISTER) {
 			write_lock(&cnic_dev_lock);
 			list_del_init(&dev->list);
@@ -2703,6 +2725,7 @@ static void cnic_release(void)
 		}
 
 		cnic_ulp_exit(dev);
+		cnic_unregister_netdev(dev);
 		list_del_init(&dev->list);
 		cnic_free_dev(dev);
 	}
-- 
1.5.6.GIT

[PATCH 3/5] bnx2: Use mutex on slow path cnic calls.

[Michael Chan]

The slow path calls to the cnic driver are sleepable calls so we
cannot use rcu_read_lock().  Use mutex for these slow path calls
instead.

Signed-off-by: Michael Chan <mchan@broadcom.com>
Reviewed-by: Benjamin Li <benli@broadcom.com>
---
 drivers/net/bnx2.c |   17 +++++++++++------
 drivers/net/bnx2.h |    1 +
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/drivers/net/bnx2.c b/drivers/net/bnx2.c
index b70cc99..06b9011 100644
--- a/drivers/net/bnx2.c
+++ b/drivers/net/bnx2.c
@@ -399,9 +399,11 @@ static int bnx2_unregister_cnic(struct net_device *dev)
 	struct bnx2_napi *bnapi = &bp->bnx2_napi[0];
 	struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
 
+	mutex_lock(&bp->cnic_lock);
 	cp->drv_state = 0;
 	bnapi->cnic_present = 0;
 	rcu_assign_pointer(bp->cnic_ops, NULL);
+	mutex_unlock(&bp->cnic_lock);
 	synchronize_rcu();
 	return 0;
 }
@@ -429,13 +431,13 @@ bnx2_cnic_stop(struct bnx2 *bp)
 	struct cnic_ops *c_ops;
 	struct cnic_ctl_info info;
 
-	rcu_read_lock();
-	c_ops = rcu_dereference(bp->cnic_ops);
+	mutex_lock(&bp->cnic_lock);
+	c_ops = bp->cnic_ops;
 	if (c_ops) {
 		info.cmd = CNIC_CTL_STOP_CMD;
 		c_ops->cnic_ctl(bp->cnic_data, &info);
 	}
-	rcu_read_unlock();
+	mutex_unlock(&bp->cnic_lock);
 }
 
 static void
@@ -444,8 +446,8 @@ bnx2_cnic_start(struct bnx2 *bp)
 	struct cnic_ops *c_ops;
 	struct cnic_ctl_info info;
 
-	rcu_read_lock();
-	c_ops = rcu_dereference(bp->cnic_ops);
+	mutex_lock(&bp->cnic_lock);
+	c_ops = bp->cnic_ops;
 	if (c_ops) {
 		if (!(bp->flags & BNX2_FLAG_USING_MSIX)) {
 			struct bnx2_napi *bnapi = &bp->bnx2_napi[0];
@@ -455,7 +457,7 @@ bnx2_cnic_start(struct bnx2 *bp)
 		info.cmd = CNIC_CTL_START_CMD;
 		c_ops->cnic_ctl(bp->cnic_data, &info);
 	}
-	rcu_read_unlock();
+	mutex_unlock(&bp->cnic_lock);
 }
 
 #else
@@ -7663,6 +7665,9 @@ bnx2_init_board(struct pci_dev *pdev, struct net_device *dev)
 
 	spin_lock_init(&bp->phy_lock);
 	spin_lock_init(&bp->indirect_lock);
+#ifdef BCM_CNIC
+	mutex_init(&bp->cnic_lock);
+#endif
 	INIT_WORK(&bp->reset_task, bnx2_reset_task);
 
 	dev->base_addr = dev->mem_start = pci_resource_start(pdev, 0);
diff --git a/drivers/net/bnx2.h b/drivers/net/bnx2.h
index f1edfaa..a4f12fd 100644
--- a/drivers/net/bnx2.h
+++ b/drivers/net/bnx2.h
@@ -6902,6 +6902,7 @@ struct bnx2 {
 	u32			idle_chk_status_idx;
 
 #ifdef BCM_CNIC
+	struct mutex		cnic_lock;
 	struct cnic_eth_dev	cnic_eth_dev;
 #endif
 
-- 
1.5.6.GIT

[PATCH 4/5] cnic: Fix locking in start/stop calls.

[Michael Chan]

The slow path ulp_start and ulp_stop calls to the bnx2i driver
are sleepable calls and therefore should not be protected using
rcu_read_lock.  Fix it by using mutex and setting a bit during
these calls.  cnic_unregister_device() will now wait for the bit
to clear before completing the call.

Signed-off-by: Michael Chan <mchan@broadcom.com>
Reviewed-by: Benjamin Li <benli@broadcom.com>
---
 drivers/net/cnic.c |   44 ++++++++++++++++++++++++++++----------------
 drivers/net/cnic.h |    1 +
 2 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/drivers/net/cnic.c b/drivers/net/cnic.c
index 2db81f0..4ff618a 100644
--- a/drivers/net/cnic.c
+++ b/drivers/net/cnic.c
@@ -466,6 +466,7 @@ EXPORT_SYMBOL(cnic_register_driver);
 static int cnic_unregister_device(struct cnic_dev *dev, int ulp_type)
 {
 	struct cnic_local *cp = dev->cnic_priv;
+	int i = 0;
 
 	if (ulp_type >= MAX_CNIC_ULP_TYPE) {
 		printk(KERN_ERR PFX "cnic_unregister_device: Bad type %d\n",
@@ -486,6 +487,15 @@ static int cnic_unregister_device(struct cnic_dev *dev, int ulp_type)
 
 	synchronize_rcu();
 
+	while (test_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[ulp_type]) &&
+	       i < 20) {
+		msleep(100);
+		i++;
+	}
+	if (test_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[ulp_type]))
+		printk(KERN_WARNING PFX "%s: Failed waiting for ULP up call"
+					" to complete.\n", dev->netdev->name);
+
 	return 0;
 }
 EXPORT_SYMBOL(cnic_unregister_driver);
@@ -1076,18 +1086,23 @@ static void cnic_ulp_stop(struct cnic_dev *dev)
 	if (cp->cnic_uinfo)
 		cnic_send_nlmsg(cp, ISCSI_KEVENT_IF_DOWN, NULL);
 
-	rcu_read_lock();
 	for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) {
 		struct cnic_ulp_ops *ulp_ops;
 
-		ulp_ops = rcu_dereference(cp->ulp_ops[if_type]);
-		if (!ulp_ops)
+		mutex_lock(&cnic_lock);
+		ulp_ops = cp->ulp_ops[if_type];
+		if (!ulp_ops) {
+			mutex_unlock(&cnic_lock);
 			continue;
+		}
+		set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]);
+		mutex_unlock(&cnic_lock);
 
 		if (test_and_clear_bit(ULP_F_START, &cp->ulp_flags[if_type]))
 			ulp_ops->cnic_stop(cp->ulp_handle[if_type]);
+
+		clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]);
 	}
-	rcu_read_unlock();
 }
 
 static void cnic_ulp_start(struct cnic_dev *dev)
@@ -1095,18 +1110,23 @@ static void cnic_ulp_start(struct cnic_dev *dev)
 	struct cnic_local *cp = dev->cnic_priv;
 	int if_type;
 
-	rcu_read_lock();
 	for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) {
 		struct cnic_ulp_ops *ulp_ops;
 
-		ulp_ops = rcu_dereference(cp->ulp_ops[if_type]);
-		if (!ulp_ops || !ulp_ops->cnic_start)
+		mutex_lock(&cnic_lock);
+		ulp_ops = cp->ulp_ops[if_type];
+		if (!ulp_ops || !ulp_ops->cnic_start) {
+			mutex_unlock(&cnic_lock);
 			continue;
+		}
+		set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]);
+		mutex_unlock(&cnic_lock);
 
 		if (!test_and_set_bit(ULP_F_START, &cp->ulp_flags[if_type]))
 			ulp_ops->cnic_start(cp->ulp_handle[if_type]);
+
+		clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]);
 	}
-	rcu_read_unlock();
 }
 
 static int cnic_ctl(void *data, struct cnic_ctl_info *info)
@@ -1116,22 +1136,18 @@ static int cnic_ctl(void *data, struct cnic_ctl_info *info)
 	switch (info->cmd) {
 	case CNIC_CTL_STOP_CMD:
 		cnic_hold(dev);
-		mutex_lock(&cnic_lock);
 
 		cnic_ulp_stop(dev);
 		cnic_stop_hw(dev);
 
-		mutex_unlock(&cnic_lock);
 		cnic_put(dev);
 		break;
 	case CNIC_CTL_START_CMD:
 		cnic_hold(dev);
-		mutex_lock(&cnic_lock);
 
 		if (!cnic_start_hw(dev))
 			cnic_ulp_start(dev);
 
-		mutex_unlock(&cnic_lock);
 		cnic_put(dev);
 		break;
 	default:
@@ -2667,10 +2683,8 @@ static int cnic_netdev_event(struct notifier_block *this, unsigned long event,
 				cnic_put(dev);
 				goto done;
 			}
-			mutex_lock(&cnic_lock);
 			if (!cnic_start_hw(dev))
 				cnic_ulp_start(dev);
-			mutex_unlock(&cnic_lock);
 		}
 
 		rcu_read_lock();
@@ -2689,10 +2703,8 @@ static int cnic_netdev_event(struct notifier_block *this, unsigned long event,
 		rcu_read_unlock();
 
 		if (event == NETDEV_GOING_DOWN) {
-			mutex_lock(&cnic_lock);
 			cnic_ulp_stop(dev);
 			cnic_stop_hw(dev);
-			mutex_unlock(&cnic_lock);
 			cnic_unregister_netdev(dev);
 		} else if (event == NETDEV_UNREGISTER) {
 			write_lock(&cnic_dev_lock);
diff --git a/drivers/net/cnic.h b/drivers/net/cnic.h
index 5192d4a..a94b302 100644
--- a/drivers/net/cnic.h
+++ b/drivers/net/cnic.h
@@ -176,6 +176,7 @@ struct cnic_local {
 	unsigned long ulp_flags[MAX_CNIC_ULP_TYPE];
 #define ULP_F_INIT	0
 #define ULP_F_START	1
+#define ULP_F_CALL_PENDING	2
 	struct cnic_ulp_ops *ulp_ops[MAX_CNIC_ULP_TYPE];
 
 	/* protected by ulp_lock */
-- 
1.5.6.GIT

[PATCH 5/5] cnic: Fix locking in init/exit calls.

[Michael Chan]

The slow path ulp_init and ulp_exit calls to the bnx2i driver
are sleepable calls and therefore should not be protected using
rcu_read_lock.  Fix it by using mutex and refcount during these
calls.  cnic_unregister_driver() will now wait for the refcount
to go to zero before completing the call.

Signed-off-by: Michael Chan <mchan@broadcom.com>
Reviewed-by: Benjamin Li <benli@broadcom.com>
---
 drivers/net/cnic.c    |   48 +++++++++++++++++++++++++++++++++++++++---------
 drivers/net/cnic_if.h |    1 +
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/drivers/net/cnic.c b/drivers/net/cnic.c
index 4ff618a..74c3429 100644
--- a/drivers/net/cnic.c
+++ b/drivers/net/cnic.c
@@ -138,6 +138,16 @@ static struct cnic_dev *cnic_from_netdev(struct net_device *netdev)
 	return NULL;
 }
 
+static inline void ulp_get(struct cnic_ulp_ops *ulp_ops)
+{
+	atomic_inc(&ulp_ops->ref_count);
+}
+
+static inline void ulp_put(struct cnic_ulp_ops *ulp_ops)
+{
+	atomic_dec(&ulp_ops->ref_count);
+}
+
 static void cnic_ctx_wr(struct cnic_dev *dev, u32 cid_addr, u32 off, u32 val)
 {
 	struct cnic_local *cp = dev->cnic_priv;
@@ -358,6 +368,7 @@ int cnic_register_driver(int ulp_type, struct cnic_ulp_ops *ulp_ops)
 	}
 	read_unlock(&cnic_dev_lock);
 
+	atomic_set(&ulp_ops->ref_count, 0);
 	rcu_assign_pointer(cnic_ulp_tbl[ulp_type], ulp_ops);
 	mutex_unlock(&cnic_lock);
 
@@ -379,6 +390,8 @@ int cnic_register_driver(int ulp_type, struct cnic_ulp_ops *ulp_ops)
 int cnic_unregister_driver(int ulp_type)
 {
 	struct cnic_dev *dev;
+	struct cnic_ulp_ops *ulp_ops;
+	int i = 0;
 
 	if (ulp_type >= MAX_CNIC_ULP_TYPE) {
 		printk(KERN_ERR PFX "cnic_unregister_driver: Bad type %d\n",
@@ -386,7 +399,8 @@ int cnic_unregister_driver(int ulp_type)
 		return -EINVAL;
 	}
 	mutex_lock(&cnic_lock);
-	if (!cnic_ulp_tbl[ulp_type]) {
+	ulp_ops = cnic_ulp_tbl[ulp_type];
+	if (!ulp_ops) {
 		printk(KERN_ERR PFX "cnic_unregister_driver: Type %d has not "
 				    "been registered\n", ulp_type);
 		goto out_unlock;
@@ -411,6 +425,14 @@ int cnic_unregister_driver(int ulp_type)
 
 	mutex_unlock(&cnic_lock);
 	synchronize_rcu();
+	while ((atomic_read(&ulp_ops->ref_count) != 0) && (i < 20)) {
+		msleep(100);
+		i++;
+	}
+
+	if (atomic_read(&ulp_ops->ref_count) != 0)
+		printk(KERN_WARNING PFX "%s: Failed waiting for ref count to go"
+					" to zero.\n", dev->netdev->name);
 	return 0;
 
 out_unlock:
@@ -1161,19 +1183,23 @@ static void cnic_ulp_init(struct cnic_dev *dev)
 	int i;
 	struct cnic_local *cp = dev->cnic_priv;
 
-	rcu_read_lock();
 	for (i = 0; i < MAX_CNIC_ULP_TYPE_EXT; i++) {
 		struct cnic_ulp_ops *ulp_ops;
 
-		ulp_ops = rcu_dereference(cnic_ulp_tbl[i]);
-		if (!ulp_ops || !ulp_ops->cnic_init)
+		mutex_lock(&cnic_lock);
+		ulp_ops = cnic_ulp_tbl[i];
+		if (!ulp_ops || !ulp_ops->cnic_init) {
+			mutex_unlock(&cnic_lock);
 			continue;
+		}
+		ulp_get(ulp_ops);
+		mutex_unlock(&cnic_lock);
 
 		if (!test_and_set_bit(ULP_F_INIT, &cp->ulp_flags[i]))
 			ulp_ops->cnic_init(dev);
 
+		ulp_put(ulp_ops);
 	}
-	rcu_read_unlock();
 }
 
 static void cnic_ulp_exit(struct cnic_dev *dev)
@@ -1181,19 +1207,23 @@ static void cnic_ulp_exit(struct cnic_dev *dev)
 	int i;
 	struct cnic_local *cp = dev->cnic_priv;
 
-	rcu_read_lock();
 	for (i = 0; i < MAX_CNIC_ULP_TYPE_EXT; i++) {
 		struct cnic_ulp_ops *ulp_ops;
 
-		ulp_ops = rcu_dereference(cnic_ulp_tbl[i]);
-		if (!ulp_ops || !ulp_ops->cnic_exit)
+		mutex_lock(&cnic_lock);
+		ulp_ops = cnic_ulp_tbl[i];
+		if (!ulp_ops || !ulp_ops->cnic_exit) {
+			mutex_unlock(&cnic_lock);
 			continue;
+		}
+		ulp_get(ulp_ops);
+		mutex_unlock(&cnic_lock);
 
 		if (test_and_clear_bit(ULP_F_INIT, &cp->ulp_flags[i]))
 			ulp_ops->cnic_exit(dev);
 
+		ulp_put(ulp_ops);
 	}
-	rcu_read_unlock();
 }
 
 static int cnic_cm_offload_pg(struct cnic_sock *csk)
diff --git a/drivers/net/cnic_if.h b/drivers/net/cnic_if.h
index d1bce27..a492357 100644
--- a/drivers/net/cnic_if.h
+++ b/drivers/net/cnic_if.h
@@ -290,6 +290,7 @@ struct cnic_ulp_ops {
 	void (*iscsi_nl_send_msg)(struct cnic_dev *dev, u32 msg_type,
 				  char *data, u16 data_size);
 	struct module *owner;
+	atomic_t ref_count;
 };
 
 extern int cnic_register_driver(int ulp_type, struct cnic_ulp_ops *ulp_ops);
-- 
1.5.6.GIT

Re: [PATCH 0/5] cnic: Bug fixes.

[David Miller]

From: "Michael Chan" <mchan@broadcom.com>
Date: Fri, 14 Aug 2009 18:49:42 -0700

> This patchset fixes a panic on ia64 and some locking issues.
> Please review.  Thanks.

Looks fine to me, applied to net-2.6