[PATCH] RDMA/core: quiesce CQ polling before device shutdown on reboot

[PATCH] RDMA/core: quiesce CQ polling before device shutdown on reboot

[Chenguang Zhao]

From: Chenguang Zhao <zhaochenguang@kylinos.cn>

When executing reboot -f in an NFS over RDMA environment, there is a 5%
probability of a crash occurring. The call trace is as follows:
 [  610.937741] Unable to handle kernel paging request at virtual address ffff3773a3e867f8
 [  610.938895] ib_srpt received unrecognized IB event 8
 [  610.952435] Mem abort info:
 [  610.977383]   ESR = 0x96000005
 [  610.984027]   Exception class = DABT (current EL), IL = 32 bits
 [  610.993574]   SET = 0, FnV = 0
 [  611.000197]   EA = 0, S1PTW = 0
 [  611.006851] Data abort info:
 [  611.013183]   ISV = 0, ISS = 0x00000005
 [  611.020435]   CM = 0, WnR = 0
 [  611.026766] swapper pgtable: 64k pages, 48-bit VAs, pgdp = 00000000489bb83a
 [  611.040104] [ffff3773a3e867f8] pgd=0000000000000000, pud=0000000000000000
 [  611.053421] Internal error: Oops: 96000005 [#1] SMP
 [  611.061809] Modules linked in: rpcsec_gss_krb5(OE) auth_rpcgss(OE) nfsv4(OE) dns_resolver enfs(OE) iptable_filter nfsv3(OE) nfs_acl(OE) nfs(OE) lockd(OE) grace fscache(OE) rfkill vfat fat rpcrdma(OE) sunrpc(OE) rdma_ucm ib_srpt ib_isert iscsi_target_mod target_core_mod ib_iser rdma_cm iw_cm ib_umad libiscsi scsi_transport_iscsi ib_ipoib ib_cm ipmi_ssif aes_ce_blk crypto_simd cryptd mlx5_ib aes_ce_cipher crct10dif_ce ib_uverbs ofpart cmdlinepart ghash_ce ses ib_core sha2_ce hi_sfc sha256_arm64 enclosure sha1_ce joydev sbsa_gwdt ipmi_si ipmi_devintf ipmi_msghandler mtd spi_dw_mmio sch_fq_codel ip_tables realtek hns3 mlx5_core megaraid_sas hclge nfit mlxfw nvme hisi_sas_v3_hw hnae3 libnvdimm hisi_sas_main nvme_core host_edma_drv dm_mirror dm_region_hash dm_log
 [  611.162725] Process kworker/70:1H (pid: 778, stack limit = 0x00000000fdde5e06)
 [  611.177485] CPU: 70 PID: 778 Comm: kworker/70:1H Kdump: loaded Tainted: G           OE     4.19.90-52.40.v2207.ky10.aarch64 #4
 [  611.196574] Source Version: 2c067295fb9b1b9b1b6693820c85b8ba78e29114
 [  611.207038] Hardware name: Huawei TaiShan 200 (Model 2280)/BC82AMDDUA, BIOS 1.89 05/20/2022
 [  611.223343] Workqueue: ib-comp-wq ib_cq_poll_work [ib_core]
 [  611.233098] pstate: 20c00009 (nzCv daif +PAN +UAO)
 [  611.242075] pc : __ib_process_cq+0x88/0xe8 [ib_core]
 [  611.251215] lr : __ib_process_cq+0x90/0xe8 [ib_core]
 [  611.260316] sp : ffffa04001e2bd20
 [  611.267776] x29: ffffa04001e2bd20 x28: ffff8040a0f6dc40
 [  611.277259] x27: 0000000000000040 x26: ffff8040a0f6dc00
 [  611.286723] x25: 0000000000010000 x24: 0000000000000010
 [  611.296109] x23: 0000000000000000 x22: 000000000000000a
 [  611.305385] x21: ffff8040a0f62000 x20: ffff8040a0f6de80
 [  611.314578] x19: ffff8040a0f6dcc0 x18: 0000000000000001
 [  611.323685] x17: 0000fffe7b3ae660 x16: ffff2b4528d7c368
 [  611.332767] x15: 0000000000000001 x14: ffff2b45299da9f8
 [  611.341778] x13: 0000000000000000 x12: 0000000000000000
 [  611.350689] x11: 00000000ffffffff x10: 0000000000000000
 [  611.359482] x9 : 000000000000000a x8 : ffff8040abe80000
 [  611.368173] x7 : 0000000000000005 x6 : 000000000000000a
 [  611.376760] x5 : 000000000000174b x4 : 000000000000000a
 [  611.385224] x3 : ffffa04001e2bd1c x2 : ffff3773a3e867f8
 [  611.393611] x1 : ffff8040a0f6dcc0 x0 : ffff8040a0f62000
 [  611.401910] Call trace:
 [  611.407252]  __ib_process_cq+0x88/0xe8 [ib_core]
 [  611.414725]  ib_cq_poll_work+0x34/0xa0 [ib_core]
 [  611.422103]  process_one_work+0x1fc/0x4a8
 [  611.428793]  worker_thread+0x50/0x4d8
 [  611.435047]  kthread+0x134/0x138
 [  611.440779]  ret_from_fork+0x10/0x18
 [  611.446770] Code: f9400262 aa1303e1 aa1503e0 b40002a2 (f9400042)
 [  611.455330] SMP: stopping secondary CPUs
 [  611.467959] Starting crashdump kernel...

On forced reboot (reboot -f), mlx5 may enter shutdown while ib-comp-wq
still polls live CQs, causing use-after-free in wr_cqe->done(). Drain
completion workqueues from the kernel reboot path before device_shutdown()
via a function pointer hook, and guard CQ processing during shutdown.

Signed-off-by: Chenguang Zhao <zhaochenguang@kylinos.cn>
---
With NFS over RDMA (rpcrdma + mlx5), reboot -f can Oops in
ib_cq_poll_work -> __ib_process_cq when calling wr_cqe->done(),
often after mlx5_core: Shutdown was called.

reboot -f skips orderly shutdown and goes directly to:

kernel_restart_prepare() -> device_shutdown() -> mlx5 shutdown
Upper layers may still hold live CQs, while ib-comp-wq keeps
polling — a use-after-free race.

Normal reboot usually works because userspace has already
torn down RDMA and called ib_free_cq().

This fixes the crash, but putting InfiniBand-specific logic
into the generic reboot path does not feel ideal. Is this
the right place, or is there a better pattern — e.g. a hook
inside ib_core, driver-level quiesce, or a generic
pre-device_shutdown() mechanism?

 drivers/infiniband/core/cq.c     | 47 +++++++++++++++++++++++++++++++-
 drivers/infiniband/core/device.c |  3 ++
 include/rdma/ib_verbs.h          |  3 ++
 kernel/reboot.c                  | 11 ++++++++
 4 files changed, 63 insertions(+), 1 deletion(-)

diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c
index 3d7b6cddd131..b64905bafeaf 100644
--- a/drivers/infiniband/core/cq.c
+++ b/drivers/infiniband/core/cq.c
@@ -9,6 +9,13 @@
 #include "core_priv.h"
 
 #include <trace/events/rdma_core.h>
+
+static atomic_t ib_cq_drain;
+
+static bool ib_cq_draining(void)
+{
+	return atomic_read(&ib_cq_drain) || system_state != SYSTEM_RUNNING;
+}
 /* Max size for shared CQ, may require tuning */
 #define IB_MAX_SHARED_CQ_SZ		4096U
 
@@ -96,6 +103,9 @@ static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *wcs,
 
 	trace_cq_process(cq);
 
+	if (ib_cq_draining())
+		return 0;
+
 	/*
 	 * budget might be (-1) if the caller does not
 	 * want to bound this call, thus we need unsigned
@@ -106,6 +116,9 @@ static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *wcs,
 		for (i = 0; i < n; i++) {
 			struct ib_wc *wc = &wcs[i];
 
+			if (ib_cq_draining())
+				return completed;
+
 			if (wc->wr_cqe)
 				wc->wr_cqe->done(cq, wc);
 			else
@@ -157,7 +170,8 @@ static int ib_poll_handler(struct irq_poll *iop, int budget)
 	completed = __ib_process_cq(cq, budget, cq->wc, IB_POLL_BATCH);
 	if (completed < budget) {
 		irq_poll_complete(&cq->iop);
-		if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) {
+		if (!ib_cq_draining() &&
+		    ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) {
 			trace_cq_reschedule(cq);
 			irq_poll_sched(&cq->iop);
 		}
@@ -171,6 +185,9 @@ static int ib_poll_handler(struct irq_poll *iop, int budget)
 
 static void ib_cq_completion_softirq(struct ib_cq *cq, void *private)
 {
+	if (ib_cq_draining())
+		return;
+
 	trace_cq_schedule(cq);
 	irq_poll_sched(&cq->iop);
 }
@@ -180,8 +197,14 @@ static void ib_cq_poll_work(struct work_struct *work)
 	struct ib_cq *cq = container_of(work, struct ib_cq, work);
 	int completed;
 
+	if (ib_cq_draining())
+		return;
+
 	completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE, cq->wc,
 				    IB_POLL_BATCH);
+	if (ib_cq_draining())
+		return;
+
 	if (completed >= IB_POLL_BUDGET_WORKQUEUE ||
 	    ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)
 		queue_work(cq->comp_wq, &cq->work);
@@ -191,6 +214,9 @@ static void ib_cq_poll_work(struct work_struct *work)
 
 static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
 {
+	if (ib_cq_draining())
+		return;
+
 	trace_cq_schedule(cq);
 	queue_work(cq->comp_wq, &cq->work);
 }
@@ -359,6 +385,25 @@ void ib_free_cq(struct ib_cq *cq)
 }
 EXPORT_SYMBOL(ib_free_cq);
 
+/**
+ * ib_drain_completion_queues - Quiesce CQ polling before device shutdown
+ *
+ * Called from the kernel reboot/poweroff path immediately before
+ * device_shutdown(), while RDMA upper layers may still hold live CQs.
+ * Stops new CQ work from being queued and waits for in-flight
+ * ib_cq_poll_work handlers to finish.
+ */
+void ib_drain_completion_queues(void)
+{
+	atomic_set(&ib_cq_drain, 1);
+
+	if (ib_comp_wq)
+		flush_workqueue(ib_comp_wq);
+	if (ib_comp_unbound_wq)
+		flush_workqueue(ib_comp_unbound_wq);
+}
+EXPORT_SYMBOL_GPL(ib_drain_completion_queues);
+
 void ib_cq_pool_cleanup(struct ib_device *dev)
 {
 	struct ib_cq *cq, *n;
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index b8193e077a74..601c58abd184 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -3105,6 +3105,8 @@ static int __init ib_core_init(void)
 
 	register_netdevice_notifier(&nb_netdevice);
 
+	ib_drain_completion_queues_fn = ib_drain_completion_queues;
+
 	return 0;
 
 err_parent:
@@ -3134,6 +3136,7 @@ static int __init ib_core_init(void)
 
 static void __exit ib_core_cleanup(void)
 {
+	ib_drain_completion_queues_fn = NULL;
 	unregister_netdevice_notifier(&nb_netdevice);
 	roce_gid_mgmt_cleanup();
 	rdma_nl_unregister(RDMA_NL_LS);
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 794746de8db0..c0b6d1cca598 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -61,6 +61,9 @@ extern struct workqueue_struct *ib_wq;
 extern struct workqueue_struct *ib_comp_wq;
 extern struct workqueue_struct *ib_comp_unbound_wq;
 
+extern void (*ib_drain_completion_queues_fn)(void);
+void ib_drain_completion_queues(void);
+
 struct ib_ucq_object;
 
 __printf(2, 3) __cold
diff --git a/kernel/reboot.c b/kernel/reboot.c
index 695c33e75efd..889e77960153 100644
--- a/kernel/reboot.c
+++ b/kernel/reboot.c
@@ -81,6 +81,9 @@ void __weak (*pm_power_off)(void);
  */
 static BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
 
+void (*ib_drain_completion_queues_fn)(void);
+EXPORT_SYMBOL_GPL(ib_drain_completion_queues_fn);
+
 /**
  *	emergency_restart - reboot the system
  *
@@ -102,6 +105,10 @@ void kernel_restart_prepare(char *cmd)
 	blocking_notifier_call_chain(&reboot_notifier_list, SYS_RESTART, cmd);
 	system_state = SYSTEM_RESTART;
 	usermodehelper_disable();
+#if IS_ENABLED(CONFIG_INFINIBAND)
+	if (ib_drain_completion_queues_fn)
+		ib_drain_completion_queues_fn();
+#endif
 	device_shutdown();
 }
 
@@ -305,6 +312,10 @@ static void kernel_shutdown_prepare(enum system_states state)
 		(state == SYSTEM_HALT) ? SYS_HALT : SYS_POWER_OFF, NULL);
 	system_state = state;
 	usermodehelper_disable();
+#if IS_ENABLED(CONFIG_INFINIBAND)
+	if (ib_drain_completion_queues_fn)
+		ib_drain_completion_queues_fn();
+#endif
 	device_shutdown();
 }
 /**
-- 
2.25.1

Re: [PATCH] RDMA/core: quiesce CQ polling before device shutdown on reboot

[Jason Gunthorpe]

On Thu, Jul 02, 2026 at 03:34:22PM +0800, Chenguang Zhao wrote:

> On forced reboot (reboot -f), mlx5 may enter shutdown while ib-comp-wq
> still polls live CQs, causing use-after-free in wr_cqe->done(). Drain
> completion workqueues from the kernel reboot path before device_shutdown()
> via a function pointer hook, and guard CQ processing during shutdown.

I think this is a bug in mlx5, it should not be destroying things
other parts of the kernel depend on during its shutdown. The real purpose
of shutdown is to DMA quite the device so the kdump kernel can recover
it and use it as the kdump NIC.

Jason

[PATCH v2] RDMA/core: quiesce CQ polling before device shutdown on reboot

[Chenguang Zhao]

From: Chenguang Zhao <zhaochenguang@kylinos.cn>

On reboot -f with NFS over RDMA, mlx5 shutdown can tear the device
down while ib-comp-wq still polls live CQs, leading to UAF in
wr_cqe->done().
Mark the device shutting down before teardown, and skip CQ poll/arm
and SYS_ERROR completion delivery while that flag is set.

Signed-off-by: Chenguang Zhao <zhaochenguang@kylinos.cn>
---
1. Set the SHUTTING_DOWN flag at the mlx5 PCI/SF shutdown entry.

2. Skip SYS_ERROR notifications during shutdown to avoid the
   internal error path re-arming CQs.

3. Make mlx5_ib_poll_cq / mlx5_ib_arm_cq return immediately while
   that flag is set, so completions are no longer delivered.

 drivers/infiniband/hw/mlx5/cq.c                       |  6 ++++++
 drivers/infiniband/hw/mlx5/main.c                     | 11 +++++++++++
 drivers/net/ethernet/mellanox/mlx5/core/health.c      |  3 +++
 drivers/net/ethernet/mellanox/mlx5/core/main.c        |  1 +
 .../net/ethernet/mellanox/mlx5/core/sf/dev/driver.c   |  1 +
 include/linux/mlx5/driver.h                           |  6 ++++++
 6 files changed, 28 insertions(+)

diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 49b4bf148a4a..584445e6d2fc 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -618,6 +618,9 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 	int soft_polled = 0;
 	int npolled;
 
+	if (mlx5_core_is_shutting_down(mdev))
+		return 0;
+
 	spin_lock_irqsave(&cq->lock, flags);
 	if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
 		/* make sure no soft wqe's are waiting */
@@ -653,6 +656,9 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 	unsigned long irq_flags;
 	int ret = 0;
 
+	if (mlx5_core_is_shutting_down(mdev))
+		return 0;
+
 	spin_lock_irqsave(&cq->lock, irq_flags);
 	if (cq->notify_flags != IB_CQ_NEXT_COMP)
 		cq->notify_flags = flags & IB_CQ_SOLICITED_MASK;
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 02809114fc79..265c95129fad 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -2976,6 +2976,9 @@ static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
 	unsigned long flags_cq;
 	unsigned long flags;
 
+	if (mlx5_core_is_shutting_down(ibdev->mdev))
+		return;
+
 	INIT_LIST_HEAD(&cq_armed_list);
 
 	/* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
@@ -3200,6 +3203,9 @@ static void mlx5_ib_handle_sys_error_event(struct work_struct *_work)
 	struct mlx5_ib_dev *ibdev = work->dev;
 	struct ib_event ibev;
 
+	if (mlx5_core_is_shutting_down(ibdev->mdev))
+		goto out;
+
 	ibev.event = IB_EVENT_DEVICE_FATAL;
 	mlx5_ib_handle_internal_error(ibdev);
 	ibev.element.port_num = (u8)(unsigned long)work->param;
@@ -3222,10 +3228,15 @@ static int mlx5_ib_sys_error_event(struct notifier_block *nb,
 				   unsigned long event, void *param)
 {
 	struct mlx5_ib_event_work *work;
+	struct mlx5_ib_dev *ibdev;
 
 	if (event != MLX5_DEV_EVENT_SYS_ERROR)
 		return NOTIFY_DONE;
 
+	ibdev = container_of(nb, struct mlx5_ib_dev, sys_error_events);
+	if (mlx5_core_is_shutting_down(ibdev->mdev))
+		return NOTIFY_OK;
+
 	work = kmalloc_obj(*work, GFP_ATOMIC);
 	if (!work)
 		return NOTIFY_DONE;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c
index aeeb136f5ebc..d9cc42c4c310 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/health.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c
@@ -202,6 +202,9 @@ static void enter_error_state(struct mlx5_core_dev *dev, bool force)
 		mlx5_cmd_flush(dev);
 	}
 
+	if (mlx5_core_is_shutting_down(dev))
+		return;
+
 	mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_SYS_ERROR, (void *)1);
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 643b4aac2033..078114bfb357 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -2192,6 +2192,7 @@ static void shutdown(struct pci_dev *pdev)
 	int err;
 
 	mlx5_core_info(dev, "Shutdown was called\n");
+	set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
 	set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
 	mlx5_drain_fw_reset(dev);
 	mlx5_drain_health_wq(dev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
index 4391ef0bab5d..6f8242bfb455 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
@@ -111,6 +111,7 @@ static void mlx5_sf_dev_shutdown(struct auxiliary_device *adev)
 	struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev);
 	struct mlx5_core_dev *mdev = sf_dev->mdev;
 
+	set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &mdev->intf_state);
 	set_bit(MLX5_BREAK_FW_WAIT, &mdev->intf_state);
 	mlx5_drain_health_wq(mdev);
 	mlx5_unload_one(mdev, false);
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index b1871c0821d0..a9efc37cd254 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -653,8 +653,14 @@ enum mlx5_device_state {
 enum mlx5_interface_state {
 	MLX5_INTERFACE_STATE_UP = BIT(0),
 	MLX5_BREAK_FW_WAIT = BIT(1),
+	MLX5_INTERFACE_STATE_SHUTTING_DOWN = BIT(2),
 };
 
+static inline bool mlx5_core_is_shutting_down(struct mlx5_core_dev *dev)
+{
+	return test_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
+}
+
 enum mlx5_pci_status {
 	MLX5_PCI_STATUS_DISABLED,
 	MLX5_PCI_STATUS_ENABLED,
-- 
2.25.1

Re: [PATCH v2] RDMA/core: quiesce CQ polling before device shutdown on reboot

[Leon Romanovsky]

On Tue, Jul 14, 2026 at 03:55:58PM +0800, Chenguang Zhao wrote:
> From: Chenguang Zhao <zhaochenguang@kylinos.cn>
> 
> On reboot -f with NFS over RDMA, mlx5 shutdown can tear the device
> down while ib-comp-wq still polls live CQs, leading to UAF in
> wr_cqe->done().
> Mark the device shutting down before teardown, and skip CQ poll/arm
> and SYS_ERROR completion delivery while that flag is set.
> 
> Signed-off-by: Chenguang Zhao <zhaochenguang@kylinos.cn>
> ---
> 1. Set the SHUTTING_DOWN flag at the mlx5 PCI/SF shutdown entry.
> 
> 2. Skip SYS_ERROR notifications during shutdown to avoid the
>    internal error path re-arming CQs.
> 
> 3. Make mlx5_ib_poll_cq / mlx5_ib_arm_cq return immediately while
>    that flag is set, so completions are no longer delivered.
> 
>  drivers/infiniband/hw/mlx5/cq.c                       |  6 ++++++
>  drivers/infiniband/hw/mlx5/main.c                     | 11 +++++++++++
>  drivers/net/ethernet/mellanox/mlx5/core/health.c      |  3 +++
>  drivers/net/ethernet/mellanox/mlx5/core/main.c        |  1 +
>  .../net/ethernet/mellanox/mlx5/core/sf/dev/driver.c   |  1 +
>  include/linux/mlx5/driver.h                           |  6 ++++++
>  6 files changed, 28 insertions(+)
> 
> diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
> index 49b4bf148a4a..584445e6d2fc 100644
> --- a/drivers/infiniband/hw/mlx5/cq.c
> +++ b/drivers/infiniband/hw/mlx5/cq.c
> @@ -618,6 +618,9 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
>  	int soft_polled = 0;
>  	int npolled;
>  
> +	if (mlx5_core_is_shutting_down(mdev))
> +		return 0;
> +

1. Send patches as standalone messages, not as replies.
2. Add the target tree to the patch subject, for example:
   [PATCH rdma-next v2]
3. This is racy; nothing prevents MLX5_INTERFACE_STATE_SHUTTING_DOWN from being set
   immediately after this check.

Thanks

>  	spin_lock_irqsave(&cq->lock, flags);
>  	if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
>  		/* make sure no soft wqe's are waiting */
> @@ -653,6 +656,9 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
>  	unsigned long irq_flags;
>  	int ret = 0;
>  
> +	if (mlx5_core_is_shutting_down(mdev))
> +		return 0;
> +
>  	spin_lock_irqsave(&cq->lock, irq_flags);
>  	if (cq->notify_flags != IB_CQ_NEXT_COMP)
>  		cq->notify_flags = flags & IB_CQ_SOLICITED_MASK;
> diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
> index 02809114fc79..265c95129fad 100644
> --- a/drivers/infiniband/hw/mlx5/main.c
> +++ b/drivers/infiniband/hw/mlx5/main.c
> @@ -2976,6 +2976,9 @@ static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
>  	unsigned long flags_cq;
>  	unsigned long flags;
>  
> +	if (mlx5_core_is_shutting_down(ibdev->mdev))
> +		return;
> +
>  	INIT_LIST_HEAD(&cq_armed_list);
>  
>  	/* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
> @@ -3200,6 +3203,9 @@ static void mlx5_ib_handle_sys_error_event(struct work_struct *_work)
>  	struct mlx5_ib_dev *ibdev = work->dev;
>  	struct ib_event ibev;
>  
> +	if (mlx5_core_is_shutting_down(ibdev->mdev))
> +		goto out;
> +
>  	ibev.event = IB_EVENT_DEVICE_FATAL;
>  	mlx5_ib_handle_internal_error(ibdev);
>  	ibev.element.port_num = (u8)(unsigned long)work->param;
> @@ -3222,10 +3228,15 @@ static int mlx5_ib_sys_error_event(struct notifier_block *nb,
>  				   unsigned long event, void *param)
>  {
>  	struct mlx5_ib_event_work *work;
> +	struct mlx5_ib_dev *ibdev;
>  
>  	if (event != MLX5_DEV_EVENT_SYS_ERROR)
>  		return NOTIFY_DONE;
>  
> +	ibdev = container_of(nb, struct mlx5_ib_dev, sys_error_events);
> +	if (mlx5_core_is_shutting_down(ibdev->mdev))
> +		return NOTIFY_OK;
> +
>  	work = kmalloc_obj(*work, GFP_ATOMIC);
>  	if (!work)
>  		return NOTIFY_DONE;
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c
> index aeeb136f5ebc..d9cc42c4c310 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c
> @@ -202,6 +202,9 @@ static void enter_error_state(struct mlx5_core_dev *dev, bool force)
>  		mlx5_cmd_flush(dev);
>  	}
>  
> +	if (mlx5_core_is_shutting_down(dev))
> +		return;
> +
>  	mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_SYS_ERROR, (void *)1);
>  }
>  
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
> index 643b4aac2033..078114bfb357 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
> @@ -2192,6 +2192,7 @@ static void shutdown(struct pci_dev *pdev)
>  	int err;
>  
>  	mlx5_core_info(dev, "Shutdown was called\n");
> +	set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
>  	set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
>  	mlx5_drain_fw_reset(dev);
>  	mlx5_drain_health_wq(dev);
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
> index 4391ef0bab5d..6f8242bfb455 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
> @@ -111,6 +111,7 @@ static void mlx5_sf_dev_shutdown(struct auxiliary_device *adev)
>  	struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev);
>  	struct mlx5_core_dev *mdev = sf_dev->mdev;
>  
> +	set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &mdev->intf_state);
>  	set_bit(MLX5_BREAK_FW_WAIT, &mdev->intf_state);
>  	mlx5_drain_health_wq(mdev);
>  	mlx5_unload_one(mdev, false);
> diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
> index b1871c0821d0..a9efc37cd254 100644
> --- a/include/linux/mlx5/driver.h
> +++ b/include/linux/mlx5/driver.h
> @@ -653,8 +653,14 @@ enum mlx5_device_state {
>  enum mlx5_interface_state {
>  	MLX5_INTERFACE_STATE_UP = BIT(0),
>  	MLX5_BREAK_FW_WAIT = BIT(1),
> +	MLX5_INTERFACE_STATE_SHUTTING_DOWN = BIT(2),
>  };
>  
> +static inline bool mlx5_core_is_shutting_down(struct mlx5_core_dev *dev)
> +{
> +	return test_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
> +}
> +
>  enum mlx5_pci_status {
>  	MLX5_PCI_STATUS_DISABLED,
>  	MLX5_PCI_STATUS_ENABLED,
> -- 
> 2.25.1
> 
>