[PATCH rdma-next 1/2] RDMA/irdma: Fix FRWR logic for PAGE_SIZE > 4K

[Jacob Moroni <jmoroni@google.com>]

The HW only supports 1G, 2M, and 4K page sizes, but some ULPs
pass PAGE_SIZE to ib_map_mr_sg(), and PAGE_SIZE can be some
multiple of 4K (64K arm kernels, etc.). The HW can still support
these scenarios, but we must force it to use 4K pages. The
max_fast_reg_page_list_len and page_cnt during MR allocation
must also be scaled to accommodate for the additional PBLEs
that will be used when ib_sg_to_pages() ends up producing
more pages than what would have otherwise been required with
the original page size. It is worth noting that this change
avoids this over-allocation/cracking in the off chance that
a 1G or 2M PAGE_SIZE is ever used; it's only applicable when
PAGE_SIZE is some power of 2 multiple of 4K that is not
currently supported by HW.

This was discovered by encountering SMMU faults when running
kernel NFS/RDMA on a 64K kernel, which was due to the page
list holding PAGE_SIZE entries while the WQE was indicating
4K, so HW was translating every access after the first 4K
incorrectly.

Fixes: b48c24c2d710 ("RDMA/irdma: Implement device supported verb APIs")
Tested-by: Leah Li <leahlijuan@google.com>
Signed-off-by: Jacob Moroni <jmoroni@google.com>
---
 drivers/infiniband/hw/irdma/ctrl.c  |  4 +++-
 drivers/infiniband/hw/irdma/defs.h  |  1 +
 drivers/infiniband/hw/irdma/verbs.c | 36 +++++++++++++++++++++++++++--
 3 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c
index 1c2ad5773b9f..ef8627f070ce 100644
--- a/drivers/infiniband/hw/irdma/ctrl.c
+++ b/drivers/infiniband/hw/irdma/ctrl.c
@@ -1703,8 +1703,10 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
 		page_size = IRDMA_PAGE_SIZE_1G;
 	else if (info->page_size == 0x200000)
 		page_size = IRDMA_PAGE_SIZE_2M;
-	else
+	else if (info->page_size == 0x1000)
 		page_size = IRDMA_PAGE_SIZE_4K;
+	else
+		return -EINVAL;
 
 	sq_info.wr_id = info->wr_id;
 	sq_info.signaled = info->signaled;
diff --git a/drivers/infiniband/hw/irdma/defs.h b/drivers/infiniband/hw/irdma/defs.h
index d6a3152959dd..eb9179664884 100644
--- a/drivers/infiniband/hw/irdma/defs.h
+++ b/drivers/infiniband/hw/irdma/defs.h
@@ -159,6 +159,7 @@ enum irdma_protocol_used {
 
 #define IRDMA_HW_PAGE_SIZE	4096
 #define IRDMA_HW_PAGE_SHIFT	12
+#define IRDMA_HW_PAGES_PER_PAGE	(PAGE_SIZE >> IRDMA_HW_PAGE_SHIFT)
 #define IRDMA_CQE_QTYPE_RQ	0
 #define IRDMA_CQE_QTYPE_SQ	1
 
diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
index c91c55917de9..9443e9732c4a 100644
--- a/drivers/infiniband/hw/irdma/verbs.c
+++ b/drivers/infiniband/hw/irdma/verbs.c
@@ -2,6 +2,21 @@
 /* Copyright (c) 2015 - 2021 Intel Corporation */
 #include "main.h"
 
+/**
+ * irdma_fmr_pages_per_page() - HW pages needed per PAGE_SIZE page in an FRWR.
+ * @dev: sc device
+ *
+ * If PAGE_SIZE is one of the HW supported page sizes, then it can be directly
+ * used. Otherwise, we need to break PAGE_SIZE into multiple HW (4K) pages.
+ */
+static u32 irdma_fmr_pages_per_page(struct irdma_sc_dev *dev)
+{
+	if (PAGE_SIZE & dev->hw_attrs.page_size_cap)
+		return 1;
+
+	return IRDMA_HW_PAGES_PER_PAGE;
+}
+
 /**
  * irdma_query_device - get device attributes
  * @ibdev: device pointer from stack
@@ -58,7 +73,8 @@ static int irdma_query_device(struct ib_device *ibdev,
 	props->max_mcast_grp = rf->max_mcg;
 	props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX;
 	props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX;
-	props->max_fast_reg_page_list_len = IRDMA_MAX_PAGES_PER_FMR;
+	props->max_fast_reg_page_list_len =
+		IRDMA_MAX_PAGES_PER_FMR / irdma_fmr_pages_per_page(&rf->sc_dev);
 	props->max_srq = rf->max_srq - rf->used_srqs;
 	props->max_srq_wr = IRDMA_MAX_SRQ_WRS;
 	props->max_srq_sge = hw_attrs->uk_attrs.max_hw_wq_frags;
@@ -3208,12 +3224,16 @@ static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
 				    u32 max_num_sg)
 {
 	struct irdma_device *iwdev = to_iwdev(pd->device);
+	u32 pages_per_page = irdma_fmr_pages_per_page(&iwdev->rf->sc_dev);
 	struct irdma_pble_alloc *palloc;
 	struct irdma_pbl *iwpbl;
 	struct irdma_mr *iwmr;
 	u32 stag;
 	int err_code;
 
+	if (max_num_sg > IRDMA_MAX_PAGES_PER_FMR / pages_per_page)
+		return ERR_PTR(-EINVAL);
+
 	iwmr = kzalloc_obj(*iwmr);
 	if (!iwmr)
 		return ERR_PTR(-ENOMEM);
@@ -3233,7 +3253,12 @@ static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
 	iwpbl->iwmr = iwmr;
 	iwmr->type = IRDMA_MEMREG_TYPE_MEM;
 	palloc = &iwpbl->pble_alloc;
-	iwmr->page_cnt = max_num_sg;
+	/* Some ULPs pass PAGE_SIZE to ib_map_mr_sg() but PAGE_SIZE can be some
+	 * multiple of 4K that the HW does not natively support. We can work
+	 * around this by cracking the larger page into multiple 4K pages, but
+	 * page_cnt needs to be scaled.
+	 */
+	iwmr->page_cnt = max_num_sg * pages_per_page;
 	/* Use system PAGE_SIZE as the sg page sizes are unknown at this point */
 	iwmr->len = max_num_sg * PAGE_SIZE;
 	err_code = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt,
@@ -3297,8 +3322,15 @@ static int irdma_set_page(struct ib_mr *ibmr, u64 addr)
 static int irdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 			   int sg_nents, unsigned int *sg_offset)
 {
+	struct irdma_device *iwdev = to_iwdev(ibmr->device);
 	struct irdma_mr *iwmr = to_iwmr(ibmr);
 
+	/* HW only supports 4K, 2M and 1G pages, but some ULPs pass PAGE_SIZE
+	 * which can be some multiple of 4K that we do not support, so force 4K.
+	 */
+	if (!(ibmr->page_size & iwdev->rf->sc_dev.hw_attrs.page_size_cap))
+		ibmr->page_size = SZ_4K;
+
 	iwmr->npages = 0;
 
 	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, irdma_set_page);
-- 
2.56.0.218.g1ec904b173-goog