[PATCH V1 rdma-next 17/24] RDMA/mlx5: Ignore completion events for a kernel CQ that is not ready

[Yishai Hadas <yishaih@nvidia.com>]

A kernel CQ can be reached from the completion interrupt before it is
set up: __ib_alloc_cq() sets its poll context and handler only after
ops.create_cq(), and devr->c0 has no handler, as it is never armed.

A trusted NIC raises no event for a CQ that is not armed, but a hostile
NIC in adversarial mode can. For devr->c0 that ends in a NULL call. For
a CQ from __ib_alloc_cq(), on a weakly ordered CPU, the interrupt can
see the handler but not the poll context, and crash in irq_poll.

In adversarial mode, give kernel CQs their own event callback, which
ignores events until the first arm, the point where the core has
finished setting up the CQ. mlx5_ib_arm_cq() marks the CQ ready with a
release before the doorbell, and the callback checks it with an
acquire.

Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
---
 drivers/infiniband/hw/mlx5/cq.c      | 31 +++++++++++++++++++++++++++-
 drivers/infiniband/hw/mlx5/mlx5_ib.h |  2 ++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 54ab85b99f45..85ae8ba311e1 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -49,6 +49,24 @@ static void mlx5_ib_cq_comp(struct mlx5_core_cq *cq, struct mlx5_eqe *eqe)
 	ibcq->comp_handler(ibcq, ibcq->cq_context);
 }
 
+/*
+ * A hostile NIC can raise an event for a kernel CQ before the core arms it,
+ * when its poll context and handler may not be visible yet. Ignore events
+ * until the core finished setting up the CQ.
+ */
+static void mlx5_ib_kernel_cq_comp(struct mlx5_core_cq *cq,
+				   struct mlx5_eqe *eqe)
+{
+	/*
+	 * Only the first events can find it unset. Pairs with the release in
+	 * mlx5_ib_arm_cq().
+	 */
+	if (!smp_load_acquire(&to_mibcq(cq)->ready))
+		return;
+
+	mlx5_ib_cq_comp(cq, eqe);
+}
+
 static void mlx5_ib_cq_event(struct mlx5_core_cq *mcq, enum mlx5_event type)
 {
 	struct mlx5_ib_cq *cq = container_of(mcq, struct mlx5_ib_cq, mcq);
@@ -835,6 +853,16 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 		ret = 1;
 	spin_unlock_irqrestore(&cq->lock, irq_flags);
 
+	/*
+	 * The core arms a kernel CQ only after its poll context and handler
+	 * are set. Mark it ready before the doorbell, so an event raised
+	 * right after it is not dropped.
+	 */
+	if (mlx5_ib_adversarial_mode && !cq->ready) {
+		/* Pairs with the acquire in mlx5_ib_kernel_cq_comp() */
+		smp_store_release(&cq->ready, true);
+	}
+
 	mlx5_cq_arm(&cq->mcq,
 		    (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
 		    MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT,
@@ -1268,7 +1296,8 @@ int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn);
 	MLX5_SET64(cqc, cqc, dbr_addr, cq->db.dma);
 
-	cq->mcq.comp = mlx5_ib_cq_comp;
+	cq->mcq.comp = mlx5_ib_adversarial_mode ? mlx5_ib_kernel_cq_comp :
+						  mlx5_ib_cq_comp;
 	/*
 	 * Set before mlx5_core_create_cq(). No real CQE can arrive before a
 	 * QP uses the CQ, but a forged one can make mlx5_ib_poll_cq() call it.
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index f7c718e3d974..44fa25ab6e0d 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -592,6 +592,8 @@ struct mlx5_ib_cq {
 	enum ib_cq_notify_flags notify_flags;
 	struct work_struct	notify_work;
 	u16			private_flags; /* Use mlx5_ib_cq_pr_flags */
+	/* adversarial_mode: kernel CQ events are handled only once set */
+	bool			ready;
 	int (*poll_one)(struct mlx5_ib_cq *cq, struct mlx5_ib_qp **cur_qp,
 			struct ib_wc *wc);
 };
-- 
2.18.1