[PATCH v21 net-next 10/12] net/nebula-matrix: dispatch: add resource ops lock protection

["illusion.wang" <illusion.wang@nebula-matrix.com>]

From: illusion wang <illusion.wang@nebula-matrix.com>

Add mutual exclusion lock for resource ops that modify shared hardware state:
1. Introduce ops_mutex_lock, init/destroy in dispatch setup/remove paths
2. NBL_OPS_CALL_LOCK_RET wrapper to wrap resource ops with lock hold
3. Wrap configure_msix_map / destroy_msix_map / set_mailbox_irq with lock
4. Fix teardown order inversion bug: unregister all channel message handlers
   before mutex destroy, eliminate potential lockdep splat or mutex corruption
5. Cleanup registered message handlers on nbl_disp_setup_msg() failure path

Depends: dispatch core routing + channel message framework patches

The read-only get_vsi_id/get_eth_id ops access static init-time data
without concurrent writer, so no lock required for them.

Signed-off-by: illusion wang <illusion.wang@nebula-matrix.com>
---
 .../nebula-matrix/nbl/nbl_core/nbl_dispatch.c | 40 ++++++++++++-------
 .../nebula-matrix/nbl/nbl_core/nbl_dispatch.h | 18 +++++++++
 2 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c
index e083dbf78543..2a04249df7a4 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c
@@ -153,9 +153,9 @@ static int nbl_disp_configure_msix_map(struct nbl_dispatch_mgt *disp_mgt,
 	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
 	struct nbl_common_info *common = disp_mgt->common;
 
-	return NBL_OPS_CALL_RET(res_ops->configure_msix_map, (p,
+	return NBL_OPS_CALL_LOCK_RET(disp_mgt, res_ops->configure_msix_map, p,
 				     common->mgt_pf, num_net_msix,
-				     num_others_msix, net_msix_mask_en));
+				     num_others_msix, net_msix_mask_en);
 }
 
 static int
@@ -195,10 +195,10 @@ static void nbl_disp_chan_configure_msix_map_resp(void *priv, u16 src_id,
 
 	copy_len = data_len < sizeof(param) ? data_len : sizeof(param);
 	memcpy(&param, data, copy_len);
-	ret = NBL_OPS_CALL_RET(res_ops->configure_msix_map,
-			       (p, src_id, le16_to_cpu(param.num_net_msix),
-			       le16_to_cpu(param.num_others_msix),
-			       le16_to_cpu(param.msix_mask_en)));
+	ret = NBL_OPS_CALL_LOCK_RET(disp_mgt, res_ops->configure_msix_map, p,
+				    src_id, le16_to_cpu(param.num_net_msix),
+				    le16_to_cpu(param.num_others_msix),
+				    le16_to_cpu(param.msix_mask_en));
 	if (ret)
 		err = NBL_CHAN_RESP_ERR;
 	if (!res_ops->configure_msix_map)
@@ -237,7 +237,8 @@ static void nbl_disp_chan_destroy_msix_map_resp(void *priv, u16 src_id,
 	int err = NBL_CHAN_RESP_OK;
 	int ret;
 
-	ret = NBL_OPS_CALL_RET(res_ops->destroy_msix_map, (p, src_id));
+	ret = NBL_OPS_CALL_LOCK_RET(disp_mgt, res_ops->destroy_msix_map, p,
+				    src_id);
 	if (ret)
 		err = NBL_CHAN_RESP_ERR;
 	if (!res_ops->destroy_msix_map)
@@ -289,8 +290,8 @@ static void nbl_disp_chan_set_mailbox_irq_resp(void *priv, u16 src_id,
 	memcpy(&param, data, copy_len);
 	vector_id = le16_to_cpu(param.vector_id);
 	enable_msix = !!param.enable_msix;
-	ret = NBL_OPS_CALL_RET(res_ops->set_mailbox_irq,
-			       (p, src_id, vector_id, enable_msix));
+	ret = NBL_OPS_CALL_LOCK_RET(disp_mgt, res_ops->set_mailbox_irq, p,
+				    src_id, vector_id, enable_msix);
 	if (ret)
 		err = NBL_CHAN_RESP_ERR;
 	if (!res_ops->set_mailbox_irq)
@@ -311,8 +312,8 @@ static int nbl_disp_destroy_msix_map(struct nbl_dispatch_mgt *disp_mgt)
 	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
 	struct nbl_common_info *common = disp_mgt->common;
 
-	return NBL_OPS_CALL_RET(res_ops->destroy_msix_map, (p,
-				     common->mgt_pf));
+	return NBL_OPS_CALL_LOCK_RET(disp_mgt, res_ops->destroy_msix_map, p,
+				     common->mgt_pf);
 }
 
 static int nbl_disp_set_mailbox_irq(struct nbl_dispatch_mgt *disp_mgt,
@@ -322,8 +323,8 @@ static int nbl_disp_set_mailbox_irq(struct nbl_dispatch_mgt *disp_mgt,
 	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
 	struct nbl_common_info *common = disp_mgt->common;
 
-	return NBL_OPS_CALL_RET(res_ops->set_mailbox_irq, (p,
-				     common->mgt_pf, vector_id, enable_msix));
+	return NBL_OPS_CALL_LOCK_RET(disp_mgt, res_ops->set_mailbox_irq, p,
+				     common->mgt_pf, vector_id, enable_msix);
 }
 
 static int nbl_disp_get_vsi_id(struct nbl_dispatch_mgt *disp_mgt, u16 type,
@@ -401,6 +402,8 @@ static int nbl_disp_setup_msg(struct nbl_dispatch_mgt *disp_mgt)
 	int ret = 0;
 	int _ret;
 
+	mutex_init(&disp_mgt->ops_mutex_lock);
+
 #define NBL_DISP_SET_OPS(disp_op, func, ctrl, msg_type, msg_req, resp) \
 do {									\
 	typeof(msg_type) _msg_type = (msg_type);			\
@@ -417,8 +420,10 @@ do {									\
 } while (0)
 	NBL_DISP_OPS_TBL;
 #undef  NBL_DISP_SET_OPS
-	if (ret)
+	if (ret) {
 		chan_ops->unregister_all_msg(p);
+		mutex_destroy(&disp_mgt->ops_mutex_lock);
+	}
 	return ret;
 }
 
@@ -515,6 +520,12 @@ int nbl_disp_init(struct nbl_adapter *adapter)
 	if (common->has_ctrl)
 		nbl_disp_setup_ctrl_lvl(disp_mgt, NBL_DISP_CTRL_LVL_MGT);
 
+	/* This bit must be set for adapters with net capability,
+	 * otherwise dispatch ops will be not set..
+	 */
+	if (common->has_net)
+		nbl_disp_setup_ctrl_lvl(disp_mgt, NBL_DISP_CTRL_LVL_NET);
+
 	return 0;
 }
 
@@ -525,4 +536,5 @@ void nbl_disp_remove(struct nbl_adapter *adapter)
 	struct nbl_channel_mgt *p = disp_mgt->chan_ops_tbl->priv;
 
 	chan_ops->unregister_all_msg(p);
+	mutex_destroy(&disp_mgt->ops_mutex_lock);
 }
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h
index f06b90075af4..722758354ff2 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h
@@ -13,12 +13,30 @@
 #include "../nbl_include/nbl_def_common.h"
 #include "../nbl_core.h"
 
+#define NBL_OPS_CALL_LOCK_RET(disp_mgt, func, ...)			\
+({									\
+	typeof(disp_mgt) _disp_mgt = (disp_mgt);			\
+	typeof(func) _func = (func);					\
+	typeof(_func(__VA_ARGS__)) _ret = 0;				\
+									\
+	mutex_lock(&_disp_mgt->ops_mutex_lock);				\
+									\
+	if (_func)							\
+		_ret = _func(__VA_ARGS__);				\
+									\
+	mutex_unlock(&_disp_mgt->ops_mutex_lock);			\
+									\
+	_ret;								\
+})
+
 struct nbl_dispatch_mgt {
 	struct nbl_common_info *common;
 	struct nbl_resource_ops_tbl *res_ops_tbl;
 	struct nbl_channel_ops_tbl *chan_ops_tbl;
 	struct nbl_dispatch_ops_tbl *disp_ops_tbl;
 	DECLARE_BITMAP(ctrl_lvl, NBL_DISP_CTRL_LVL_MAX);
+	/* use for the caller not in interrupt */
+	struct mutex ops_mutex_lock;
 };
 
 #endif
-- 
2.47.3