Re: [PATCH v2] RDMA/core: quiesce CQ polling before device shutdown on reboot

[Leon Romanovsky <leon@kernel.org>]

On Tue, Jul 14, 2026 at 03:55:58PM +0800, Chenguang Zhao wrote:
> From: Chenguang Zhao <zhaochenguang@kylinos.cn>
> 
> On reboot -f with NFS over RDMA, mlx5 shutdown can tear the device
> down while ib-comp-wq still polls live CQs, leading to UAF in
> wr_cqe->done().
> Mark the device shutting down before teardown, and skip CQ poll/arm
> and SYS_ERROR completion delivery while that flag is set.
> 
> Signed-off-by: Chenguang Zhao <zhaochenguang@kylinos.cn>
> ---
> 1. Set the SHUTTING_DOWN flag at the mlx5 PCI/SF shutdown entry.
> 
> 2. Skip SYS_ERROR notifications during shutdown to avoid the
>    internal error path re-arming CQs.
> 
> 3. Make mlx5_ib_poll_cq / mlx5_ib_arm_cq return immediately while
>    that flag is set, so completions are no longer delivered.
> 
>  drivers/infiniband/hw/mlx5/cq.c                       |  6 ++++++
>  drivers/infiniband/hw/mlx5/main.c                     | 11 +++++++++++
>  drivers/net/ethernet/mellanox/mlx5/core/health.c      |  3 +++
>  drivers/net/ethernet/mellanox/mlx5/core/main.c        |  1 +
>  .../net/ethernet/mellanox/mlx5/core/sf/dev/driver.c   |  1 +
>  include/linux/mlx5/driver.h                           |  6 ++++++
>  6 files changed, 28 insertions(+)
> 
> diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
> index 49b4bf148a4a..584445e6d2fc 100644
> --- a/drivers/infiniband/hw/mlx5/cq.c
> +++ b/drivers/infiniband/hw/mlx5/cq.c
> @@ -618,6 +618,9 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
>  	int soft_polled = 0;
>  	int npolled;
>  
> +	if (mlx5_core_is_shutting_down(mdev))
> +		return 0;
> +

1. Send patches as standalone messages, not as replies.
2. Add the target tree to the patch subject, for example:
   [PATCH rdma-next v2]
3. This is racy; nothing prevents MLX5_INTERFACE_STATE_SHUTTING_DOWN from being set
   immediately after this check.

Thanks

>  	spin_lock_irqsave(&cq->lock, flags);
>  	if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
>  		/* make sure no soft wqe's are waiting */
> @@ -653,6 +656,9 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
>  	unsigned long irq_flags;
>  	int ret = 0;
>  
> +	if (mlx5_core_is_shutting_down(mdev))
> +		return 0;
> +
>  	spin_lock_irqsave(&cq->lock, irq_flags);
>  	if (cq->notify_flags != IB_CQ_NEXT_COMP)
>  		cq->notify_flags = flags & IB_CQ_SOLICITED_MASK;
> diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
> index 02809114fc79..265c95129fad 100644
> --- a/drivers/infiniband/hw/mlx5/main.c
> +++ b/drivers/infiniband/hw/mlx5/main.c
> @@ -2976,6 +2976,9 @@ static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
>  	unsigned long flags_cq;
>  	unsigned long flags;
>  
> +	if (mlx5_core_is_shutting_down(ibdev->mdev))
> +		return;
> +
>  	INIT_LIST_HEAD(&cq_armed_list);
>  
>  	/* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
> @@ -3200,6 +3203,9 @@ static void mlx5_ib_handle_sys_error_event(struct work_struct *_work)
>  	struct mlx5_ib_dev *ibdev = work->dev;
>  	struct ib_event ibev;
>  
> +	if (mlx5_core_is_shutting_down(ibdev->mdev))
> +		goto out;
> +
>  	ibev.event = IB_EVENT_DEVICE_FATAL;
>  	mlx5_ib_handle_internal_error(ibdev);
>  	ibev.element.port_num = (u8)(unsigned long)work->param;
> @@ -3222,10 +3228,15 @@ static int mlx5_ib_sys_error_event(struct notifier_block *nb,
>  				   unsigned long event, void *param)
>  {
>  	struct mlx5_ib_event_work *work;
> +	struct mlx5_ib_dev *ibdev;
>  
>  	if (event != MLX5_DEV_EVENT_SYS_ERROR)
>  		return NOTIFY_DONE;
>  
> +	ibdev = container_of(nb, struct mlx5_ib_dev, sys_error_events);
> +	if (mlx5_core_is_shutting_down(ibdev->mdev))
> +		return NOTIFY_OK;
> +
>  	work = kmalloc_obj(*work, GFP_ATOMIC);
>  	if (!work)
>  		return NOTIFY_DONE;
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c
> index aeeb136f5ebc..d9cc42c4c310 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c
> @@ -202,6 +202,9 @@ static void enter_error_state(struct mlx5_core_dev *dev, bool force)
>  		mlx5_cmd_flush(dev);
>  	}
>  
> +	if (mlx5_core_is_shutting_down(dev))
> +		return;
> +
>  	mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_SYS_ERROR, (void *)1);
>  }
>  
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
> index 643b4aac2033..078114bfb357 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
> @@ -2192,6 +2192,7 @@ static void shutdown(struct pci_dev *pdev)
>  	int err;
>  
>  	mlx5_core_info(dev, "Shutdown was called\n");
> +	set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
>  	set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
>  	mlx5_drain_fw_reset(dev);
>  	mlx5_drain_health_wq(dev);
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
> index 4391ef0bab5d..6f8242bfb455 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
> @@ -111,6 +111,7 @@ static void mlx5_sf_dev_shutdown(struct auxiliary_device *adev)
>  	struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev);
>  	struct mlx5_core_dev *mdev = sf_dev->mdev;
>  
> +	set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &mdev->intf_state);
>  	set_bit(MLX5_BREAK_FW_WAIT, &mdev->intf_state);
>  	mlx5_drain_health_wq(mdev);
>  	mlx5_unload_one(mdev, false);
> diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
> index b1871c0821d0..a9efc37cd254 100644
> --- a/include/linux/mlx5/driver.h
> +++ b/include/linux/mlx5/driver.h
> @@ -653,8 +653,14 @@ enum mlx5_device_state {
>  enum mlx5_interface_state {
>  	MLX5_INTERFACE_STATE_UP = BIT(0),
>  	MLX5_BREAK_FW_WAIT = BIT(1),
> +	MLX5_INTERFACE_STATE_SHUTTING_DOWN = BIT(2),
>  };
>  
> +static inline bool mlx5_core_is_shutting_down(struct mlx5_core_dev *dev)
> +{
> +	return test_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
> +}
> +
>  enum mlx5_pci_status {
>  	MLX5_PCI_STATUS_DISABLED,
>  	MLX5_PCI_STATUS_ENABLED,
> -- 
> 2.25.1
> 
>