* [PATCH v2] RDMA/core: quiesce CQ polling before device shutdown on reboot
[not found] <20260702073422.279820-1-chenguang.zhao@linux.dev>
@ 2026-07-14 7:55 ` Chenguang Zhao
2026-07-14 8:46 ` Leon Romanovsky
0 siblings, 1 reply; 2+ messages in thread
From: Chenguang Zhao @ 2026-07-14 7:55 UTC (permalink / raw)
To: leon, jgg, saeedm, tariqt, mbloch, davem, edumazet, kuba, pabeni
Cc: linux-rdma, netdev, chenguang.zhao, Chenguang Zhao
From: Chenguang Zhao <zhaochenguang@kylinos.cn>
On reboot -f with NFS over RDMA, mlx5 shutdown can tear the device
down while ib-comp-wq still polls live CQs, leading to UAF in
wr_cqe->done().
Mark the device shutting down before teardown, and skip CQ poll/arm
and SYS_ERROR completion delivery while that flag is set.
Signed-off-by: Chenguang Zhao <zhaochenguang@kylinos.cn>
---
1. Set the SHUTTING_DOWN flag at the mlx5 PCI/SF shutdown entry.
2. Skip SYS_ERROR notifications during shutdown to avoid the
internal error path re-arming CQs.
3. Make mlx5_ib_poll_cq / mlx5_ib_arm_cq return immediately while
that flag is set, so completions are no longer delivered.
drivers/infiniband/hw/mlx5/cq.c | 6 ++++++
drivers/infiniband/hw/mlx5/main.c | 11 +++++++++++
drivers/net/ethernet/mellanox/mlx5/core/health.c | 3 +++
drivers/net/ethernet/mellanox/mlx5/core/main.c | 1 +
.../net/ethernet/mellanox/mlx5/core/sf/dev/driver.c | 1 +
include/linux/mlx5/driver.h | 6 ++++++
6 files changed, 28 insertions(+)
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 49b4bf148a4a..584445e6d2fc 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -618,6 +618,9 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
int soft_polled = 0;
int npolled;
+ if (mlx5_core_is_shutting_down(mdev))
+ return 0;
+
spin_lock_irqsave(&cq->lock, flags);
if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
/* make sure no soft wqe's are waiting */
@@ -653,6 +656,9 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
unsigned long irq_flags;
int ret = 0;
+ if (mlx5_core_is_shutting_down(mdev))
+ return 0;
+
spin_lock_irqsave(&cq->lock, irq_flags);
if (cq->notify_flags != IB_CQ_NEXT_COMP)
cq->notify_flags = flags & IB_CQ_SOLICITED_MASK;
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 02809114fc79..265c95129fad 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -2976,6 +2976,9 @@ static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
unsigned long flags_cq;
unsigned long flags;
+ if (mlx5_core_is_shutting_down(ibdev->mdev))
+ return;
+
INIT_LIST_HEAD(&cq_armed_list);
/* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
@@ -3200,6 +3203,9 @@ static void mlx5_ib_handle_sys_error_event(struct work_struct *_work)
struct mlx5_ib_dev *ibdev = work->dev;
struct ib_event ibev;
+ if (mlx5_core_is_shutting_down(ibdev->mdev))
+ goto out;
+
ibev.event = IB_EVENT_DEVICE_FATAL;
mlx5_ib_handle_internal_error(ibdev);
ibev.element.port_num = (u8)(unsigned long)work->param;
@@ -3222,10 +3228,15 @@ static int mlx5_ib_sys_error_event(struct notifier_block *nb,
unsigned long event, void *param)
{
struct mlx5_ib_event_work *work;
+ struct mlx5_ib_dev *ibdev;
if (event != MLX5_DEV_EVENT_SYS_ERROR)
return NOTIFY_DONE;
+ ibdev = container_of(nb, struct mlx5_ib_dev, sys_error_events);
+ if (mlx5_core_is_shutting_down(ibdev->mdev))
+ return NOTIFY_OK;
+
work = kmalloc_obj(*work, GFP_ATOMIC);
if (!work)
return NOTIFY_DONE;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c
index aeeb136f5ebc..d9cc42c4c310 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/health.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c
@@ -202,6 +202,9 @@ static void enter_error_state(struct mlx5_core_dev *dev, bool force)
mlx5_cmd_flush(dev);
}
+ if (mlx5_core_is_shutting_down(dev))
+ return;
+
mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_SYS_ERROR, (void *)1);
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 643b4aac2033..078114bfb357 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -2192,6 +2192,7 @@ static void shutdown(struct pci_dev *pdev)
int err;
mlx5_core_info(dev, "Shutdown was called\n");
+ set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
mlx5_drain_fw_reset(dev);
mlx5_drain_health_wq(dev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
index 4391ef0bab5d..6f8242bfb455 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
@@ -111,6 +111,7 @@ static void mlx5_sf_dev_shutdown(struct auxiliary_device *adev)
struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev);
struct mlx5_core_dev *mdev = sf_dev->mdev;
+ set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &mdev->intf_state);
set_bit(MLX5_BREAK_FW_WAIT, &mdev->intf_state);
mlx5_drain_health_wq(mdev);
mlx5_unload_one(mdev, false);
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index b1871c0821d0..a9efc37cd254 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -653,8 +653,14 @@ enum mlx5_device_state {
enum mlx5_interface_state {
MLX5_INTERFACE_STATE_UP = BIT(0),
MLX5_BREAK_FW_WAIT = BIT(1),
+ MLX5_INTERFACE_STATE_SHUTTING_DOWN = BIT(2),
};
+static inline bool mlx5_core_is_shutting_down(struct mlx5_core_dev *dev)
+{
+ return test_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
+}
+
enum mlx5_pci_status {
MLX5_PCI_STATUS_DISABLED,
MLX5_PCI_STATUS_ENABLED,
--
2.25.1
^ permalink raw reply related [flat|nested] 2+ messages in thread
* Re: [PATCH v2] RDMA/core: quiesce CQ polling before device shutdown on reboot
2026-07-14 7:55 ` [PATCH v2] RDMA/core: quiesce CQ polling before device shutdown on reboot Chenguang Zhao
@ 2026-07-14 8:46 ` Leon Romanovsky
0 siblings, 0 replies; 2+ messages in thread
From: Leon Romanovsky @ 2026-07-14 8:46 UTC (permalink / raw)
To: Chenguang Zhao
Cc: jgg, saeedm, tariqt, mbloch, davem, edumazet, kuba, pabeni,
linux-rdma, netdev, Chenguang Zhao
On Tue, Jul 14, 2026 at 03:55:58PM +0800, Chenguang Zhao wrote:
> From: Chenguang Zhao <zhaochenguang@kylinos.cn>
>
> On reboot -f with NFS over RDMA, mlx5 shutdown can tear the device
> down while ib-comp-wq still polls live CQs, leading to UAF in
> wr_cqe->done().
> Mark the device shutting down before teardown, and skip CQ poll/arm
> and SYS_ERROR completion delivery while that flag is set.
>
> Signed-off-by: Chenguang Zhao <zhaochenguang@kylinos.cn>
> ---
> 1. Set the SHUTTING_DOWN flag at the mlx5 PCI/SF shutdown entry.
>
> 2. Skip SYS_ERROR notifications during shutdown to avoid the
> internal error path re-arming CQs.
>
> 3. Make mlx5_ib_poll_cq / mlx5_ib_arm_cq return immediately while
> that flag is set, so completions are no longer delivered.
>
> drivers/infiniband/hw/mlx5/cq.c | 6 ++++++
> drivers/infiniband/hw/mlx5/main.c | 11 +++++++++++
> drivers/net/ethernet/mellanox/mlx5/core/health.c | 3 +++
> drivers/net/ethernet/mellanox/mlx5/core/main.c | 1 +
> .../net/ethernet/mellanox/mlx5/core/sf/dev/driver.c | 1 +
> include/linux/mlx5/driver.h | 6 ++++++
> 6 files changed, 28 insertions(+)
>
> diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
> index 49b4bf148a4a..584445e6d2fc 100644
> --- a/drivers/infiniband/hw/mlx5/cq.c
> +++ b/drivers/infiniband/hw/mlx5/cq.c
> @@ -618,6 +618,9 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
> int soft_polled = 0;
> int npolled;
>
> + if (mlx5_core_is_shutting_down(mdev))
> + return 0;
> +
1. Send patches as standalone messages, not as replies.
2. Add the target tree to the patch subject, for example:
[PATCH rdma-next v2]
3. This is racy; nothing prevents MLX5_INTERFACE_STATE_SHUTTING_DOWN from being set
immediately after this check.
Thanks
> spin_lock_irqsave(&cq->lock, flags);
> if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
> /* make sure no soft wqe's are waiting */
> @@ -653,6 +656,9 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
> unsigned long irq_flags;
> int ret = 0;
>
> + if (mlx5_core_is_shutting_down(mdev))
> + return 0;
> +
> spin_lock_irqsave(&cq->lock, irq_flags);
> if (cq->notify_flags != IB_CQ_NEXT_COMP)
> cq->notify_flags = flags & IB_CQ_SOLICITED_MASK;
> diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
> index 02809114fc79..265c95129fad 100644
> --- a/drivers/infiniband/hw/mlx5/main.c
> +++ b/drivers/infiniband/hw/mlx5/main.c
> @@ -2976,6 +2976,9 @@ static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
> unsigned long flags_cq;
> unsigned long flags;
>
> + if (mlx5_core_is_shutting_down(ibdev->mdev))
> + return;
> +
> INIT_LIST_HEAD(&cq_armed_list);
>
> /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
> @@ -3200,6 +3203,9 @@ static void mlx5_ib_handle_sys_error_event(struct work_struct *_work)
> struct mlx5_ib_dev *ibdev = work->dev;
> struct ib_event ibev;
>
> + if (mlx5_core_is_shutting_down(ibdev->mdev))
> + goto out;
> +
> ibev.event = IB_EVENT_DEVICE_FATAL;
> mlx5_ib_handle_internal_error(ibdev);
> ibev.element.port_num = (u8)(unsigned long)work->param;
> @@ -3222,10 +3228,15 @@ static int mlx5_ib_sys_error_event(struct notifier_block *nb,
> unsigned long event, void *param)
> {
> struct mlx5_ib_event_work *work;
> + struct mlx5_ib_dev *ibdev;
>
> if (event != MLX5_DEV_EVENT_SYS_ERROR)
> return NOTIFY_DONE;
>
> + ibdev = container_of(nb, struct mlx5_ib_dev, sys_error_events);
> + if (mlx5_core_is_shutting_down(ibdev->mdev))
> + return NOTIFY_OK;
> +
> work = kmalloc_obj(*work, GFP_ATOMIC);
> if (!work)
> return NOTIFY_DONE;
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c
> index aeeb136f5ebc..d9cc42c4c310 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c
> @@ -202,6 +202,9 @@ static void enter_error_state(struct mlx5_core_dev *dev, bool force)
> mlx5_cmd_flush(dev);
> }
>
> + if (mlx5_core_is_shutting_down(dev))
> + return;
> +
> mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_SYS_ERROR, (void *)1);
> }
>
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
> index 643b4aac2033..078114bfb357 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
> @@ -2192,6 +2192,7 @@ static void shutdown(struct pci_dev *pdev)
> int err;
>
> mlx5_core_info(dev, "Shutdown was called\n");
> + set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
> set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
> mlx5_drain_fw_reset(dev);
> mlx5_drain_health_wq(dev);
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
> index 4391ef0bab5d..6f8242bfb455 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
> @@ -111,6 +111,7 @@ static void mlx5_sf_dev_shutdown(struct auxiliary_device *adev)
> struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev);
> struct mlx5_core_dev *mdev = sf_dev->mdev;
>
> + set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &mdev->intf_state);
> set_bit(MLX5_BREAK_FW_WAIT, &mdev->intf_state);
> mlx5_drain_health_wq(mdev);
> mlx5_unload_one(mdev, false);
> diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
> index b1871c0821d0..a9efc37cd254 100644
> --- a/include/linux/mlx5/driver.h
> +++ b/include/linux/mlx5/driver.h
> @@ -653,8 +653,14 @@ enum mlx5_device_state {
> enum mlx5_interface_state {
> MLX5_INTERFACE_STATE_UP = BIT(0),
> MLX5_BREAK_FW_WAIT = BIT(1),
> + MLX5_INTERFACE_STATE_SHUTTING_DOWN = BIT(2),
> };
>
> +static inline bool mlx5_core_is_shutting_down(struct mlx5_core_dev *dev)
> +{
> + return test_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
> +}
> +
> enum mlx5_pci_status {
> MLX5_PCI_STATUS_DISABLED,
> MLX5_PCI_STATUS_ENABLED,
> --
> 2.25.1
>
>
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2026-07-14 8:46 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
[not found] <20260702073422.279820-1-chenguang.zhao@linux.dev>
2026-07-14 7:55 ` [PATCH v2] RDMA/core: quiesce CQ polling before device shutdown on reboot Chenguang Zhao
2026-07-14 8:46 ` Leon Romanovsky
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox