* Re: [PATCH v2] RDMA/core: quiesce CQ polling before device shutdown on reboot
2026-07-14 7:55 ` [PATCH v2] RDMA/core: quiesce CQ polling before device shutdown on reboot Chenguang Zhao
@ 2026-07-14 8:46 ` Leon Romanovsky
0 siblings, 0 replies; 2+ messages in thread
From: Leon Romanovsky @ 2026-07-14 8:46 UTC (permalink / raw)
To: Chenguang Zhao
Cc: jgg, saeedm, tariqt, mbloch, davem, edumazet, kuba, pabeni,
linux-rdma, netdev, Chenguang Zhao
On Tue, Jul 14, 2026 at 03:55:58PM +0800, Chenguang Zhao wrote:
> From: Chenguang Zhao <zhaochenguang@kylinos.cn>
>
> On reboot -f with NFS over RDMA, mlx5 shutdown can tear the device
> down while ib-comp-wq still polls live CQs, leading to UAF in
> wr_cqe->done().
> Mark the device shutting down before teardown, and skip CQ poll/arm
> and SYS_ERROR completion delivery while that flag is set.
>
> Signed-off-by: Chenguang Zhao <zhaochenguang@kylinos.cn>
> ---
> 1. Set the SHUTTING_DOWN flag at the mlx5 PCI/SF shutdown entry.
>
> 2. Skip SYS_ERROR notifications during shutdown to avoid the
> internal error path re-arming CQs.
>
> 3. Make mlx5_ib_poll_cq / mlx5_ib_arm_cq return immediately while
> that flag is set, so completions are no longer delivered.
>
> drivers/infiniband/hw/mlx5/cq.c | 6 ++++++
> drivers/infiniband/hw/mlx5/main.c | 11 +++++++++++
> drivers/net/ethernet/mellanox/mlx5/core/health.c | 3 +++
> drivers/net/ethernet/mellanox/mlx5/core/main.c | 1 +
> .../net/ethernet/mellanox/mlx5/core/sf/dev/driver.c | 1 +
> include/linux/mlx5/driver.h | 6 ++++++
> 6 files changed, 28 insertions(+)
>
> diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
> index 49b4bf148a4a..584445e6d2fc 100644
> --- a/drivers/infiniband/hw/mlx5/cq.c
> +++ b/drivers/infiniband/hw/mlx5/cq.c
> @@ -618,6 +618,9 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
> int soft_polled = 0;
> int npolled;
>
> + if (mlx5_core_is_shutting_down(mdev))
> + return 0;
> +
1. Send patches as standalone messages, not as replies.
2. Add the target tree to the patch subject, for example:
[PATCH rdma-next v2]
3. This is racy; nothing prevents MLX5_INTERFACE_STATE_SHUTTING_DOWN from being set
immediately after this check.
Thanks
> spin_lock_irqsave(&cq->lock, flags);
> if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
> /* make sure no soft wqe's are waiting */
> @@ -653,6 +656,9 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
> unsigned long irq_flags;
> int ret = 0;
>
> + if (mlx5_core_is_shutting_down(mdev))
> + return 0;
> +
> spin_lock_irqsave(&cq->lock, irq_flags);
> if (cq->notify_flags != IB_CQ_NEXT_COMP)
> cq->notify_flags = flags & IB_CQ_SOLICITED_MASK;
> diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
> index 02809114fc79..265c95129fad 100644
> --- a/drivers/infiniband/hw/mlx5/main.c
> +++ b/drivers/infiniband/hw/mlx5/main.c
> @@ -2976,6 +2976,9 @@ static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
> unsigned long flags_cq;
> unsigned long flags;
>
> + if (mlx5_core_is_shutting_down(ibdev->mdev))
> + return;
> +
> INIT_LIST_HEAD(&cq_armed_list);
>
> /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
> @@ -3200,6 +3203,9 @@ static void mlx5_ib_handle_sys_error_event(struct work_struct *_work)
> struct mlx5_ib_dev *ibdev = work->dev;
> struct ib_event ibev;
>
> + if (mlx5_core_is_shutting_down(ibdev->mdev))
> + goto out;
> +
> ibev.event = IB_EVENT_DEVICE_FATAL;
> mlx5_ib_handle_internal_error(ibdev);
> ibev.element.port_num = (u8)(unsigned long)work->param;
> @@ -3222,10 +3228,15 @@ static int mlx5_ib_sys_error_event(struct notifier_block *nb,
> unsigned long event, void *param)
> {
> struct mlx5_ib_event_work *work;
> + struct mlx5_ib_dev *ibdev;
>
> if (event != MLX5_DEV_EVENT_SYS_ERROR)
> return NOTIFY_DONE;
>
> + ibdev = container_of(nb, struct mlx5_ib_dev, sys_error_events);
> + if (mlx5_core_is_shutting_down(ibdev->mdev))
> + return NOTIFY_OK;
> +
> work = kmalloc_obj(*work, GFP_ATOMIC);
> if (!work)
> return NOTIFY_DONE;
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c
> index aeeb136f5ebc..d9cc42c4c310 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c
> @@ -202,6 +202,9 @@ static void enter_error_state(struct mlx5_core_dev *dev, bool force)
> mlx5_cmd_flush(dev);
> }
>
> + if (mlx5_core_is_shutting_down(dev))
> + return;
> +
> mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_SYS_ERROR, (void *)1);
> }
>
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
> index 643b4aac2033..078114bfb357 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
> @@ -2192,6 +2192,7 @@ static void shutdown(struct pci_dev *pdev)
> int err;
>
> mlx5_core_info(dev, "Shutdown was called\n");
> + set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
> set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
> mlx5_drain_fw_reset(dev);
> mlx5_drain_health_wq(dev);
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
> index 4391ef0bab5d..6f8242bfb455 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
> @@ -111,6 +111,7 @@ static void mlx5_sf_dev_shutdown(struct auxiliary_device *adev)
> struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev);
> struct mlx5_core_dev *mdev = sf_dev->mdev;
>
> + set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &mdev->intf_state);
> set_bit(MLX5_BREAK_FW_WAIT, &mdev->intf_state);
> mlx5_drain_health_wq(mdev);
> mlx5_unload_one(mdev, false);
> diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
> index b1871c0821d0..a9efc37cd254 100644
> --- a/include/linux/mlx5/driver.h
> +++ b/include/linux/mlx5/driver.h
> @@ -653,8 +653,14 @@ enum mlx5_device_state {
> enum mlx5_interface_state {
> MLX5_INTERFACE_STATE_UP = BIT(0),
> MLX5_BREAK_FW_WAIT = BIT(1),
> + MLX5_INTERFACE_STATE_SHUTTING_DOWN = BIT(2),
> };
>
> +static inline bool mlx5_core_is_shutting_down(struct mlx5_core_dev *dev)
> +{
> + return test_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
> +}
> +
> enum mlx5_pci_status {
> MLX5_PCI_STATUS_DISABLED,
> MLX5_PCI_STATUS_ENABLED,
> --
> 2.25.1
>
>
^ permalink raw reply [flat|nested] 2+ messages in thread