From: Yishai Hadas <yishaih@nvidia.com>
To: <alex.williamson@redhat.com>, <jgg@nvidia.com>
Cc: <saeedm@nvidia.com>, <kvm@vger.kernel.org>,
<netdev@vger.kernel.org>, <kuba@kernel.org>,
<kevin.tian@intel.com>, <joao.m.martins@oracle.com>,
<leonro@nvidia.com>, <yishaih@nvidia.com>, <maorg@nvidia.com>,
<cohuck@redhat.com>
Subject: [PATCH V7 vfio 09/10] vfio/mlx5: Manage error scenarios on tracker
Date: Thu, 8 Sep 2022 21:34:47 +0300 [thread overview]
Message-ID: <20220908183448.195262-10-yishaih@nvidia.com> (raw)
In-Reply-To: <20220908183448.195262-1-yishaih@nvidia.com>
Handle async error events and health/recovery flow to safely stop the
tracker upon error scenarios.
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
---
drivers/vfio/pci/mlx5/cmd.c | 61 +++++++++++++++++++++++++++++++++++--
drivers/vfio/pci/mlx5/cmd.h | 2 ++
2 files changed, 61 insertions(+), 2 deletions(-)
diff --git a/drivers/vfio/pci/mlx5/cmd.c b/drivers/vfio/pci/mlx5/cmd.c
index fa9ddd926500..3e92b4d92be2 100644
--- a/drivers/vfio/pci/mlx5/cmd.c
+++ b/drivers/vfio/pci/mlx5/cmd.c
@@ -70,6 +70,13 @@ int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev,
return 0;
}
+static void set_tracker_error(struct mlx5vf_pci_core_device *mvdev)
+{
+ /* Mark the tracker under an error and wake it up if it's running */
+ mvdev->tracker.is_err = true;
+ complete(&mvdev->tracker_comp);
+}
+
static int mlx5fv_vf_event(struct notifier_block *nb,
unsigned long event, void *data)
{
@@ -100,6 +107,8 @@ void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev)
if (!mvdev->migrate_cap)
return;
+ /* Must be done outside the lock to let it progress */
+ set_tracker_error(mvdev);
mutex_lock(&mvdev->state_mutex);
mlx5vf_disable_fds(mvdev);
_mlx5vf_free_page_tracker_resources(mvdev);
@@ -619,6 +628,47 @@ static void mlx5vf_destroy_cq(struct mlx5_core_dev *mdev,
mlx5_db_free(mdev, &cq->db);
}
+static void mlx5vf_cq_event(struct mlx5_core_cq *mcq, enum mlx5_event type)
+{
+ if (type != MLX5_EVENT_TYPE_CQ_ERROR)
+ return;
+
+ set_tracker_error(container_of(mcq, struct mlx5vf_pci_core_device,
+ tracker.cq.mcq));
+}
+
+static int mlx5vf_event_notifier(struct notifier_block *nb, unsigned long type,
+ void *data)
+{
+ struct mlx5_vhca_page_tracker *tracker =
+ mlx5_nb_cof(nb, struct mlx5_vhca_page_tracker, nb);
+ struct mlx5vf_pci_core_device *mvdev = container_of(
+ tracker, struct mlx5vf_pci_core_device, tracker);
+ struct mlx5_eqe *eqe = data;
+ u8 event_type = (u8)type;
+ u8 queue_type;
+ int qp_num;
+
+ switch (event_type) {
+ case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
+ case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
+ case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
+ queue_type = eqe->data.qp_srq.type;
+ if (queue_type != MLX5_EVENT_QUEUE_TYPE_QP)
+ break;
+ qp_num = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
+ if (qp_num != tracker->host_qp->qpn &&
+ qp_num != tracker->fw_qp->qpn)
+ break;
+ set_tracker_error(mvdev);
+ break;
+ default:
+ break;
+ }
+
+ return NOTIFY_OK;
+}
+
static void mlx5vf_cq_complete(struct mlx5_core_cq *mcq,
struct mlx5_eqe *eqe)
{
@@ -680,6 +730,7 @@ static int mlx5vf_create_cq(struct mlx5_core_dev *mdev,
pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas);
mlx5_fill_page_frag_array(&cq->buf.frag_buf, pas);
cq->mcq.comp = mlx5vf_cq_complete;
+ cq->mcq.event = mlx5vf_cq_event;
err = mlx5_core_create_cq(mdev, &cq->mcq, in, inlen, out, sizeof(out));
if (err)
goto err_vec;
@@ -1014,6 +1065,7 @@ _mlx5vf_free_page_tracker_resources(struct mlx5vf_pci_core_device *mvdev)
WARN_ON(mvdev->mdev_detach);
+ mlx5_eq_notifier_unregister(mdev, &tracker->nb);
mlx5vf_cmd_destroy_tracker(mdev, tracker->id);
mlx5vf_destroy_qp(mdev, tracker->fw_qp);
mlx5vf_free_qp_recv_resources(mdev, tracker->host_qp);
@@ -1127,6 +1179,8 @@ int mlx5vf_start_page_tracker(struct vfio_device *vdev,
if (err)
goto err_activate;
+ MLX5_NB_INIT(&tracker->nb, mlx5vf_event_notifier, NOTIFY_ANY);
+ mlx5_eq_notifier_register(mdev, &tracker->nb);
*page_size = host_qp->tracked_page_size;
mvdev->log_active = true;
mlx5vf_state_mutex_unlock(mvdev);
@@ -1273,7 +1327,8 @@ int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova,
goto end;
tracker->status = MLX5_PAGE_TRACK_STATE_REPORTING;
- while (tracker->status == MLX5_PAGE_TRACK_STATE_REPORTING) {
+ while (tracker->status == MLX5_PAGE_TRACK_STATE_REPORTING &&
+ !tracker->is_err) {
poll_err = mlx5vf_cq_poll_one(cq, tracker->host_qp, dirty,
&tracker->status);
if (poll_err == CQ_EMPTY) {
@@ -1294,8 +1349,10 @@ int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova,
}
if (tracker->status == MLX5_PAGE_TRACK_STATE_ERROR)
- err = -EIO;
+ tracker->is_err = true;
+ if (tracker->is_err)
+ err = -EIO;
end:
mlx5vf_state_mutex_unlock(mvdev);
return err;
diff --git a/drivers/vfio/pci/mlx5/cmd.h b/drivers/vfio/pci/mlx5/cmd.h
index fa1f9ab4d3d0..8b0ae40c620c 100644
--- a/drivers/vfio/pci/mlx5/cmd.h
+++ b/drivers/vfio/pci/mlx5/cmd.h
@@ -82,10 +82,12 @@ struct mlx5_vhca_qp {
struct mlx5_vhca_page_tracker {
u32 id;
u32 pdn;
+ u8 is_err:1;
struct mlx5_uars_page *uar;
struct mlx5_vhca_cq cq;
struct mlx5_vhca_qp *host_qp;
struct mlx5_vhca_qp *fw_qp;
+ struct mlx5_nb nb;
int status;
};
--
2.18.1
next prev parent reply other threads:[~2022-09-08 18:36 UTC|newest]
Thread overview: 21+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-09-08 18:34 [PATCH V7 vfio 00/10] Add device DMA logging support for mlx5 driver Yishai Hadas
2022-09-08 18:34 ` [PATCH V7 vfio 01/10] net/mlx5: Introduce ifc bits for page tracker Yishai Hadas
2022-09-08 18:34 ` [PATCH V7 vfio 02/10] net/mlx5: Query ADV_VIRTUALIZATION capabilities Yishai Hadas
2022-09-08 18:34 ` [PATCH V7 vfio 03/10] vfio: Introduce DMA logging uAPIs Yishai Hadas
2022-09-08 18:34 ` [PATCH V7 vfio 04/10] vfio: Add an IOVA bitmap support Yishai Hadas
2022-09-08 18:34 ` [PATCH V7 vfio 05/10] vfio: Introduce the DMA logging feature support Yishai Hadas
2022-09-08 18:34 ` [PATCH V7 vfio 06/10] vfio/mlx5: Init QP based resources for dirty tracking Yishai Hadas
2022-09-08 18:34 ` [PATCH V7 vfio 07/10] vfio/mlx5: Create and destroy page tracker object Yishai Hadas
2023-09-06 8:55 ` Cédric Le Goater
2023-09-06 9:48 ` Yishai Hadas
2023-09-06 11:51 ` Jason Gunthorpe
2023-09-06 12:08 ` Joao Martins
2023-09-07 9:56 ` Cédric Le Goater
2023-09-07 10:51 ` Joao Martins
2023-09-07 12:16 ` Cédric Le Goater
2023-09-07 16:33 ` Joao Martins
2023-09-07 17:34 ` Cédric Le Goater
2022-09-08 18:34 ` [PATCH V7 vfio 08/10] vfio/mlx5: Report dirty pages from tracker Yishai Hadas
2022-09-08 18:34 ` Yishai Hadas [this message]
2022-09-08 18:34 ` [PATCH V7 vfio 10/10] vfio/mlx5: Set the driver DMA logging callbacks Yishai Hadas
2022-09-08 20:17 ` [PATCH V7 vfio 00/10] Add device DMA logging support for mlx5 driver Alex Williamson
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220908183448.195262-10-yishaih@nvidia.com \
--to=yishaih@nvidia.com \
--cc=alex.williamson@redhat.com \
--cc=cohuck@redhat.com \
--cc=jgg@nvidia.com \
--cc=joao.m.martins@oracle.com \
--cc=kevin.tian@intel.com \
--cc=kuba@kernel.org \
--cc=kvm@vger.kernel.org \
--cc=leonro@nvidia.com \
--cc=maorg@nvidia.com \
--cc=netdev@vger.kernel.org \
--cc=saeedm@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).