[PATCH iwl-net 1/3] ice: avoid RTNL deadlock when notifying the RDMA auxiliary driver

[Petr Oros <poros@redhat.com>]

ice_send_event_to_aux() blocks on the auxiliary device lock, and the
dcbnl callers reach it with the RTNL lock held. The RDMA auxiliary probe
takes the two in the opposite order, because __driver_attach() holds the
auxiliary device lock across ib_register_device(), which needs the RTNL
lock:

  systemd-udevd   state:D
    really_probe                  <- holds device_lock(&adev->dev)
    icrdma_probe
    irdma_ib_register_device
    ib_register_device
    setup_device
    irdma_roce_port_immutable
    ib_query_port
    irdma_query_port
    ib_get_eth_speed              <- blocks on rtnl_lock()

  lldpad          state:D
    rtnetlink_rcv_msg             <- holds rtnl_lock()
    dcb_doit
    dcbnl_ieee_set
    ice_dcbnl_setets
    ice_pf_dcb_cfg
    ice_send_event_to_aux         <- blocks on device_lock(&adev->dev)

Everything else that needs the RTNL lock piles up behind lldpad and the
boot never completes. The probe side cannot be changed, so use
device_trylock() and report whether the event reached the driver.

IIDC_RDMA_EVENT_BEFORE_TC_CHANGE has to arrive before the traffic
classes change, so ice_pf_dcb_cfg() refuses the request with -EBUSY
rather than run it behind the RDMA driver's back. The critical error and
MTU change events are sent from the service task, which consumes their
pending bits before sending, so those are re-armed instead.
AFTER_TC_CHANGE keeps its fire and forget behaviour, it can be neither
refused, the hardware has already changed, nor deferred, every lock that
would serialise a deferred delivery against the dcbnl callers forms
another cycle.

Fixes: 348048e724a0 ("ice: Implement iidc operations")
Cc: stable@vger.kernel.org
Signed-off-by: Petr Oros <poros@redhat.com>
---
 drivers/net/ethernet/intel/ice/ice_dcb_lib.c |  9 ++++++++-
 drivers/net/ethernet/intel/ice/ice_idc.c     | 20 ++++++++++++++++----
 drivers/net/ethernet/intel/ice/ice_idc_int.h |  2 +-
 drivers/net/ethernet/intel/ice/ice_main.c    | 16 ++++++++++++----
 4 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c
index 0bc6dd37568792..2022b46c990cb2 100644
--- a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c
@@ -355,8 +355,10 @@ int ice_pf_dcb_cfg(struct ice_pf *pf, struct ice_dcbx_cfg *new_cfg, bool locked)
 	struct iidc_rdma_event *event;
 	int ret = ICE_DCB_NO_HW_CHG;
 	struct ice_vsi *pf_vsi;
+	bool dcb_ena;
 
 	curr_cfg = &pf->hw.port_info->qos_cfg.local_dcbx_cfg;
+	dcb_ena = test_bit(ICE_FLAG_DCB_ENA, pf->flags);
 
 	/* FW does not care if change happened */
 	if (!pf->hw.port_info->qos_cfg.is_sw_lldp)
@@ -406,7 +408,12 @@ int ice_pf_dcb_cfg(struct ice_pf *pf, struct ice_dcbx_cfg *new_cfg, bool locked)
 	}
 
 	set_bit(IIDC_RDMA_EVENT_BEFORE_TC_CHANGE, event->type);
-	ice_send_event_to_aux(pf, event);
+	if (!ice_send_event_to_aux(pf, event)) {
+		assign_bit(ICE_FLAG_DCB_ENA, pf->flags, dcb_ena);
+		kfree(event);
+		ret = -EBUSY;
+		goto free_cfg;
+	}
 	kfree(event);
 
 	/* avoid race conditions by holding the lock while disabling and
diff --git a/drivers/net/ethernet/intel/ice/ice_idc.c b/drivers/net/ethernet/intel/ice/ice_idc.c
index 102d63c3018b1f..7fd35ce966ea65 100644
--- a/drivers/net/ethernet/intel/ice/ice_idc.c
+++ b/drivers/net/ethernet/intel/ice/ice_idc.c
@@ -34,30 +34,42 @@ ice_get_auxiliary_drv(struct iidc_rdma_core_dev_info *cdev)
  * ice_send_event_to_aux - send event to RDMA AUX driver
  * @pf: pointer to PF struct
  * @event: event struct
+ *
+ * Return: false when the auxiliary device lock was contended.
  */
-void ice_send_event_to_aux(struct ice_pf *pf, struct iidc_rdma_event *event)
+bool ice_send_event_to_aux(struct ice_pf *pf, struct iidc_rdma_event *event)
 {
 	struct iidc_rdma_core_auxiliary_drv *iadrv;
 	struct iidc_rdma_core_dev_info *cdev;
+	bool sent = true;
 
 	if (WARN_ON_ONCE(!in_task()))
-		return;
+		return true;
 
 	cdev = pf->cdev_info;
 	if (!cdev)
-		return;
+		return true;
 
 	mutex_lock(&pf->adev_mutex);
 	if (!cdev->adev)
 		goto finish;
 
-	device_lock(&cdev->adev->dev);
+	if (!device_trylock(&cdev->adev->dev)) {
+		dev_dbg(ice_pf_to_dev(pf),
+			"auxiliary device busy, dropped RDMA event 0x%lx\n",
+			*event->type);
+		sent = false;
+		goto finish;
+	}
+
 	iadrv = ice_get_auxiliary_drv(cdev);
 	if (iadrv && iadrv->event_handler)
 		iadrv->event_handler(cdev, event);
 	device_unlock(&cdev->adev->dev);
 finish:
 	mutex_unlock(&pf->adev_mutex);
+
+	return sent;
 }
 
 /**
diff --git a/drivers/net/ethernet/intel/ice/ice_idc_int.h b/drivers/net/ethernet/intel/ice/ice_idc_int.h
index 17dbfcfb6a2aff..e7d249d6140d92 100644
--- a/drivers/net/ethernet/intel/ice/ice_idc_int.h
+++ b/drivers/net/ethernet/intel/ice/ice_idc_int.h
@@ -9,6 +9,6 @@
 
 struct ice_pf;
 
-void ice_send_event_to_aux(struct ice_pf *pf, struct iidc_rdma_event *event);
+bool ice_send_event_to_aux(struct ice_pf *pf, struct iidc_rdma_event *event);
 
 #endif /* !_ICE_IDC_INT_H_ */
diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
index d88835482d3aac..edb62606d2aae3 100644
--- a/drivers/net/ethernet/intel/ice/ice_main.c
+++ b/drivers/net/ethernet/intel/ice/ice_main.c
@@ -2313,11 +2313,16 @@ static void ice_service_task(struct work_struct *work)
 		struct iidc_rdma_event *event;
 
 		event = kzalloc_obj(*event);
-		if (event) {
+		if (!event) {
+			set_bit(ICE_AUX_ERR_PENDING, pf->state);
+		} else {
 			set_bit(IIDC_RDMA_EVENT_CRIT_ERR, event->type);
 			/* report the entire OICR value to AUX driver */
 			swap(event->reg, pf->oicr_err_reg);
-			ice_send_event_to_aux(pf, event);
+			if (!ice_send_event_to_aux(pf, event)) {
+				pf->oicr_err_reg |= event->reg;
+				set_bit(ICE_AUX_ERR_PENDING, pf->state);
+			}
 			kfree(event);
 		}
 	}
@@ -2336,9 +2341,12 @@ static void ice_service_task(struct work_struct *work)
 		struct iidc_rdma_event *event;
 
 		event = kzalloc_obj(*event);
-		if (event) {
+		if (!event) {
+			set_bit(ICE_FLAG_MTU_CHANGED, pf->flags);
+		} else {
 			set_bit(IIDC_RDMA_EVENT_AFTER_MTU_CHANGE, event->type);
-			ice_send_event_to_aux(pf, event);
+			if (!ice_send_event_to_aux(pf, event))
+				set_bit(ICE_FLAG_MTU_CHANGED, pf->flags);
 			kfree(event);
 		}
 	}
-- 
2.56.0