[PATCH v2 for-next 0/7] RDMA/hns: Add more restrack attributes to hns driver

[PATCH v2 for-next 0/7] RDMA/hns: Add more restrack attributes to hns driver

[Wenpeng Liang]

The hns driver already supports CQ's restrack ops. Therefore, this patchset
supports QP/MR's restrack ops and CQ/QP/MR's restrack raw ops. Among them,
restrack ops dump the queue information maintained by the driver, and
restrack raw ops dump the queue context maintained by ROCEE.

Changes since v1:
* Add support for restrack raw ops.
* Add the result dumped by rdmatool in the commit message.
* v1 Link: https://patchwork.kernel.org/project/linux-rdma/cover/20220124124624.55352-1-liangwenpeng@huawei.com/

Wenpeng Liang (7):
  RDMA/hns: Remove redundant DFX file and DFX ops structure
  RDMA/hns: Add or remove CQ's restrack attributes
  RDMA/hns: Support CQ's restrack raw ops for hns driver
  RDMA/hns: Support QP's restrack ops for hns driver
  RDMA/hns: Support QP's restrack raw ops for hns driver
  RDMA/hns: Support MR's restrack ops for hns driver
  RDMA/hns: Support MR's restrack raw ops for hns driver

 drivers/infiniband/hw/hns/Makefile            |   2 +-
 drivers/infiniband/hw/hns/hns_roce_device.h   |  17 +-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c    |  76 +++++-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h    |   6 +-
 .../infiniband/hw/hns/hns_roce_hw_v2_dfx.c    |  34 ---
 drivers/infiniband/hw/hns/hns_roce_main.c     |  11 +-
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 220 ++++++++++++++----
 7 files changed, 263 insertions(+), 103 deletions(-)
 delete mode 100644 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c

--
2.30.0

[PATCH v2 for-next 1/7] RDMA/hns: Remove redundant DFX file and DFX ops structure

[Wenpeng Liang]

There is no need to use a dedicated DXF file and DFX structure to manage
the interface of the query queue context.

Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
---
 drivers/infiniband/hw/hns/Makefile            |  2 +-
 drivers/infiniband/hw/hns/hns_roce_device.h   | 10 ++----
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c    | 35 ++++++++++++++++---
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h    |  3 --
 .../infiniband/hw/hns/hns_roce_hw_v2_dfx.c    | 34 ------------------
 drivers/infiniband/hw/hns/hns_roce_main.c     |  6 +++-
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 35 +++++++------------
 7 files changed, 50 insertions(+), 75 deletions(-)
 delete mode 100644 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c

diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile
index 9f04f25d9631..a7d259238305 100644
--- a/drivers/infiniband/hw/hns/Makefile
+++ b/drivers/infiniband/hw/hns/Makefile
@@ -10,6 +10,6 @@ hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \
 	hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o
 
 ifdef CONFIG_INFINIBAND_HNS_HIP08
-hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o $(hns-roce-objs)
+hns-roce-hw-v2-objs := hns_roce_hw_v2.o $(hns-roce-objs)
 obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v2.o
 endif
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index f848eedc6a23..103d50564b89 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -849,11 +849,6 @@ struct hns_roce_caps {
 	enum cong_type	cong_type;
 };
 
-struct hns_roce_dfx_hw {
-	int (*query_cqc_info)(struct hns_roce_dev *hr_dev, u32 cqn,
-			      int *buffer);
-};
-
 enum hns_roce_device_state {
 	HNS_ROCE_DEVICE_STATE_INITED,
 	HNS_ROCE_DEVICE_STATE_RST_DOWN,
@@ -899,6 +894,7 @@ struct hns_roce_hw {
 	int (*init_eq)(struct hns_roce_dev *hr_dev);
 	void (*cleanup_eq)(struct hns_roce_dev *hr_dev);
 	int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);
+	int (*query_cqc)(struct hns_roce_dev *hr_dev, u32 cqn, void *buffer);
 	const struct ib_device_ops *hns_roce_dev_ops;
 	const struct ib_device_ops *hns_roce_dev_srq_ops;
 };
@@ -960,7 +956,6 @@ struct hns_roce_dev {
 	void			*priv;
 	struct workqueue_struct *irq_workq;
 	struct work_struct ecc_work;
-	const struct hns_roce_dfx_hw *dfx;
 	u32 func_num;
 	u32 is_vf;
 	u32 cong_algo_tmpl_id;
@@ -1228,8 +1223,7 @@ u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u32 port, int gid_index);
 void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev);
 int hns_roce_init(struct hns_roce_dev *hr_dev);
 void hns_roce_exit(struct hns_roce_dev *hr_dev);
-int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
-			       struct ib_cq *ib_cq);
+int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index cbdafaac678a..979cd57a72fb 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -5774,6 +5774,35 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
 	return ret;
 }
 
+static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn,
+				 void *buffer)
+{
+	struct hns_roce_v2_cq_context *context;
+	struct hns_roce_cmd_mailbox *mailbox;
+	int ret;
+
+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+	if (IS_ERR(mailbox))
+		return PTR_ERR(mailbox);
+
+	context = mailbox->buf;
+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma,
+				HNS_ROCE_CMD_QUERY_CQC, cqn);
+	if (ret) {
+		ibdev_err(&hr_dev->ib_dev,
+			  "failed to process cmd when querying CQ, ret = %d.\n",
+			  ret);
+		goto err_mailbox;
+	}
+
+	memcpy(buffer, context, sizeof(*context));
+
+err_mailbox:
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+
+	return ret;
+}
+
 static void hns_roce_irq_work_handle(struct work_struct *work)
 {
 	struct hns_roce_work *irq_work =
@@ -6575,10 +6604,6 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
 	kfree(eq_table->eq);
 }
 
-static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
-	.query_cqc_info = hns_roce_v2_query_cqc_info,
-};
-
 static const struct ib_device_ops hns_roce_v2_dev_ops = {
 	.destroy_qp = hns_roce_v2_destroy_qp,
 	.modify_cq = hns_roce_v2_modify_cq,
@@ -6619,6 +6644,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {
 	.init_eq = hns_roce_v2_init_eq_table,
 	.cleanup_eq = hns_roce_v2_cleanup_eq_table,
 	.write_srqc = hns_roce_v2_write_srqc,
+	.query_cqc = hns_roce_v2_query_cqc,
 	.hns_roce_dev_ops = &hns_roce_v2_dev_ops,
 	.hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
 };
@@ -6650,7 +6676,6 @@ static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
 	hr_dev->is_vf = id->driver_data;
 	hr_dev->dev = &handle->pdev->dev;
 	hr_dev->hw = &hns_roce_hw_v2;
-	hr_dev->dfx = &hns_roce_dfx_hw_v2;
 	hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
 	hr_dev->odb_offset = hr_dev->sdb_offset;
 
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
index f96debac30fe..49ec29973ed7 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
@@ -1462,9 +1462,6 @@ struct hns_roce_sccc_clr_done {
 	__le32 rsv[5];
 };
 
-int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
-			       int *buffer);
-
 static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2],
 				    void __iomem *dest)
 {
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
deleted file mode 100644
index f7a75a7cda74..000000000000
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
+++ /dev/null
@@ -1,34 +0,0 @@
-// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
-// Copyright (c) 2019 Hisilicon Limited.
-
-#include "hnae3.h"
-#include "hns_roce_device.h"
-#include "hns_roce_cmd.h"
-#include "hns_roce_hw_v2.h"
-
-int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
-			       int *buffer)
-{
-	struct hns_roce_v2_cq_context *cq_context;
-	struct hns_roce_cmd_mailbox *mailbox;
-	int ret;
-
-	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
-	if (IS_ERR(mailbox))
-		return PTR_ERR(mailbox);
-
-	cq_context = mailbox->buf;
-	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_CQC,
-				cqn);
-	if (ret) {
-		dev_err(hr_dev->dev, "QUERY cqc cmd process error\n");
-		goto err_mailbox;
-	}
-
-	memcpy(buffer, cq_context, sizeof(*cq_context));
-
-err_mailbox:
-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
-
-	return ret;
-}
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index c8af4ebd7cbd..caf73e8f4bbe 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -515,7 +515,6 @@ static const struct ib_device_ops hns_roce_dev_ops = {
 	.destroy_ah = hns_roce_destroy_ah,
 	.destroy_cq = hns_roce_destroy_cq,
 	.disassociate_ucontext = hns_roce_disassociate_ucontext,
-	.fill_res_cq_entry = hns_roce_fill_res_cq_entry,
 	.get_dma_mr = hns_roce_get_dma_mr,
 	.get_link_layer = hns_roce_get_link_layer,
 	.get_port_immutable = hns_roce_port_immutable,
@@ -566,6 +565,10 @@ static const struct ib_device_ops hns_roce_dev_xrcd_ops = {
 	INIT_RDMA_OBJ_SIZE(ib_xrcd, hns_roce_xrcd, ibxrcd),
 };
 
+static const struct ib_device_ops hns_roce_dev_restrack_ops = {
+	.fill_res_cq_entry = hns_roce_fill_res_cq_entry,
+};
+
 static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
 {
 	int ret;
@@ -605,6 +608,7 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
 
 	ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_ops);
 	ib_set_device_ops(ib_dev, &hns_roce_dev_ops);
+	ib_set_device_ops(ib_dev, &hns_roce_dev_restrack_ops);
 	for (i = 0; i < hr_dev->caps.num_ports; i++) {
 		if (!hr_dev->iboe.netdevs[i])
 			continue;
diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c
index 24a154d64630..83417be15d3f 100644
--- a/drivers/infiniband/hw/hns/hns_roce_restrack.c
+++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c
@@ -55,45 +55,34 @@ static int hns_roce_fill_cq(struct sk_buff *msg,
 	return -EMSGSIZE;
 }
 
-int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
-			       struct ib_cq *ib_cq)
+int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
 	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
-	struct hns_roce_v2_cq_context *context;
+	struct hns_roce_v2_cq_context context;
 	struct nlattr *table_attr;
 	int ret;
 
-	if (!hr_dev->dfx->query_cqc_info)
+	if (!hr_dev->hw->query_cqc)
 		return -EINVAL;
 
-	context = kzalloc(sizeof(struct hns_roce_v2_cq_context), GFP_KERNEL);
-	if (!context)
-		return -ENOMEM;
-
-	ret = hr_dev->dfx->query_cqc_info(hr_dev, hr_cq->cqn, (int *)context);
+	ret = hr_dev->hw->query_cqc(hr_dev, hr_cq->cqn, &context);
 	if (ret)
-		goto err;
+		return -EINVAL;
 
 	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
-	if (!table_attr) {
-		ret = -EMSGSIZE;
-		goto err;
-	}
+	if (!table_attr)
+		return -EMSGSIZE;
 
-	if (hns_roce_fill_cq(msg, context)) {
-		ret = -EMSGSIZE;
-		goto err_cancel_table;
-	}
+	if (hns_roce_fill_cq(msg, &context))
+		goto err;
 
 	nla_nest_end(msg, table_attr);
-	kfree(context);
 
 	return 0;
 
-err_cancel_table:
-	nla_nest_cancel(msg, table_attr);
 err:
-	kfree(context);
-	return ret;
+	nla_nest_cancel(msg, table_attr);
+
+	return -EMSGSIZE;
 }
-- 
2.30.0

[PATCH v2 for-next 2/7] RDMA/hns: Add or remove CQ's restrack attributes

[Wenpeng Liang]

Remove the resttrack attributes from the queue context held by ROCEE, and
add the resttrack attributes from the queue information maintained by the
driver.

For example:

$ rdma res show cq dev hns_0 cqn 14 -dd -jp
[ {
        "ifindex": 4,
        "ifname": "hns_0",
        "cqn": 14,
        "cqe": 127,
        "users": 1,
        "adaptive-moderation": false,
        "ctxn": 8,
        "pid": 1524,
        "comm": "ib_send_bw"
    },
    "drv_cq_depth": 128,
    "drv_cons_index": 0,
    "drv_cqe_size": 32,
    "drv_arm_sn": 1
}

Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
---
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 67 +++----------------
 1 file changed, 10 insertions(+), 57 deletions(-)

diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c
index 83417be15d3f..2e8299784bc2 100644
--- a/drivers/infiniband/hw/hns/hns_roce_restrack.c
+++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c
@@ -9,72 +9,25 @@
 #include "hns_roce_device.h"
 #include "hns_roce_hw_v2.h"
 
-static int hns_roce_fill_cq(struct sk_buff *msg,
-			    struct hns_roce_v2_cq_context *context)
-{
-	if (rdma_nl_put_driver_u32(msg, "state",
-				   hr_reg_read(context, CQC_ARM_ST)))
-
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "ceqn",
-				   hr_reg_read(context, CQC_CEQN)))
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "cqn",
-				   hr_reg_read(context, CQC_CQN)))
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "hopnum",
-				   hr_reg_read(context, CQC_CQE_HOP_NUM)))
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "pi",
-				   hr_reg_read(context, CQC_CQ_PRODUCER_IDX)))
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "ci",
-				   hr_reg_read(context, CQC_CQ_CONSUMER_IDX)))
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "coalesce",
-				   hr_reg_read(context, CQC_CQ_MAX_CNT)))
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "period",
-				   hr_reg_read(context, CQC_CQ_PERIOD)))
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "cnt",
-				   hr_reg_read(context, CQC_CQE_CNT)))
-		goto err;
-
-	return 0;
-
-err:
-	return -EMSGSIZE;
-}
-
 int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq)
 {
-	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
 	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
-	struct hns_roce_v2_cq_context context;
 	struct nlattr *table_attr;
-	int ret;
-
-	if (!hr_dev->hw->query_cqc)
-		return -EINVAL;
-
-	ret = hr_dev->hw->query_cqc(hr_dev, hr_cq->cqn, &context);
-	if (ret)
-		return -EINVAL;
 
 	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
 	if (!table_attr)
 		return -EMSGSIZE;
 
-	if (hns_roce_fill_cq(msg, &context))
+	if (rdma_nl_put_driver_u32(msg, "cq_depth", hr_cq->cq_depth))
+		goto err;
+
+	if (rdma_nl_put_driver_u32(msg, "cons_index", hr_cq->cons_index))
+		goto err;
+
+	if (rdma_nl_put_driver_u32(msg, "cqe_size", hr_cq->cqe_size))
+		goto err;
+
+	if (rdma_nl_put_driver_u32(msg, "arm_sn", hr_cq->arm_sn))
 		goto err;
 
 	nla_nest_end(msg, table_attr);
-- 
2.30.0

[PATCH v2 for-next 3/7] RDMA/hns: Support CQ's restrack raw ops for hns driver

[Wenpeng Liang]

The CQ raw restrack attributes come from the queue context maintained by
the ROCEE.

For example:

$ rdma res show cq dev hns_0 cqn 14 -dd -jp -r
[ {
        "ifindex": 4,
        "ifname": "hns_0",
        "data": [ 1,0,0,0,7,0,0,0,0,0,0,0,0,82,6,0,0,82,6,0,0,82,6,0,
		  1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,
		  6,0,0,0,0,0,0,0 ]
    } ]

Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
---
 drivers/infiniband/hw/hns/hns_roce_device.h   |  1 +
 drivers/infiniband/hw/hns/hns_roce_main.c     |  1 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 39 +++++++++++++++++++
 3 files changed, 41 insertions(+)

diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index 103d50564b89..c73adc0d3555 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -1224,6 +1224,7 @@ void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev);
 int hns_roce_init(struct hns_roce_dev *hr_dev);
 void hns_roce_exit(struct hns_roce_dev *hr_dev);
 int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);
+int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index caf73e8f4bbe..1b66ed45350e 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -567,6 +567,7 @@ static const struct ib_device_ops hns_roce_dev_xrcd_ops = {
 
 static const struct ib_device_ops hns_roce_dev_restrack_ops = {
 	.fill_res_cq_entry = hns_roce_fill_res_cq_entry,
+	.fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw,
 };
 
 static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c
index 2e8299784bc2..3f9c2f9dfdf6 100644
--- a/drivers/infiniband/hw/hns/hns_roce_restrack.c
+++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c
@@ -9,6 +9,8 @@
 #include "hns_roce_device.h"
 #include "hns_roce_hw_v2.h"
 
+#define MAX_ENTRY_NUM 256
+
 int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq)
 {
 	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
@@ -39,3 +41,40 @@ int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq)
 
 	return -EMSGSIZE;
 }
+
+int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
+	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
+	struct hns_roce_v2_cq_context context;
+	u32 data[MAX_ENTRY_NUM] = {};
+	int offset = 0;
+	int ret;
+
+	if (!hr_dev->hw->query_cqc)
+		return -EINVAL;
+
+	ret = hr_dev->hw->query_cqc(hr_dev, hr_cq->cqn, &context);
+	if (ret)
+		return -EINVAL;
+
+	data[offset++] = hr_reg_read(&context, CQC_CQ_ST);
+	data[offset++] = hr_reg_read(&context, CQC_SHIFT);
+	data[offset++] = hr_reg_read(&context, CQC_CQE_SIZE);
+	data[offset++] = hr_reg_read(&context, CQC_CQE_CNT);
+	data[offset++] = hr_reg_read(&context, CQC_CQ_PRODUCER_IDX);
+	data[offset++] = hr_reg_read(&context, CQC_CQ_CONSUMER_IDX);
+	data[offset++] = hr_reg_read(&context, CQC_DB_RECORD_EN);
+	data[offset++] = hr_reg_read(&context, CQC_ARM_ST);
+	data[offset++] = hr_reg_read(&context, CQC_CMD_SN);
+	data[offset++] = hr_reg_read(&context, CQC_CEQN);
+	data[offset++] = hr_reg_read(&context, CQC_CQ_MAX_CNT);
+	data[offset++] = hr_reg_read(&context, CQC_CQ_PERIOD);
+	data[offset++] = hr_reg_read(&context, CQC_CQE_HOP_NUM);
+	data[offset++] = hr_reg_read(&context, CQC_CQE_BAR_PG_SZ);
+	data[offset++] = hr_reg_read(&context, CQC_CQE_BUF_PG_SZ);
+
+	ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, offset * sizeof(u32), data);
+
+	return ret;
+}
-- 
2.30.0

[PATCH v2 for-next 4/7] RDMA/hns: Support QP's restrack ops for hns driver

[Wenpeng Liang]

The QP restrack attributes come from the queue information maintained by
the driver.

For example:

$ rdma res show qp link hns_0 lqpn 41 -jp -dd
[ {
        "ifindex": 4,
        "ifname": "hns_0",
        "port": 1,
        "lqpn": 41,
        "rqpn": 40,
        "type": "RC",
        "state": "RTR",
        "rq-psn": 12474738,
        "sq-psn": 0,
        "path-mig-state": "ARMED",
        "pdn": 9,
        "pid": 1523,
        "comm": "ib_send_bw"
    },
    "drv_sq_wqe_cnt": 128,
    "drv_sq_max_gs": 1,
    "drv_rq_wqe_cnt": 512,
    "drv_rq_max_gs": 2,
    "drv_ext_sge_sge_cnt": 0
}

Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
---
 drivers/infiniband/hw/hns/hns_roce_device.h   |  1 +
 drivers/infiniband/hw/hns/hns_roce_main.c     |  1 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 34 +++++++++++++++++++
 3 files changed, 36 insertions(+)

diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index c73adc0d3555..7578c0c6313b 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -1225,6 +1225,7 @@ int hns_roce_init(struct hns_roce_dev *hr_dev);
 void hns_roce_exit(struct hns_roce_dev *hr_dev);
 int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);
 int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);
+int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index 1b66ed45350e..87442027b808 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -568,6 +568,7 @@ static const struct ib_device_ops hns_roce_dev_xrcd_ops = {
 static const struct ib_device_ops hns_roce_dev_restrack_ops = {
 	.fill_res_cq_entry = hns_roce_fill_res_cq_entry,
 	.fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw,
+	.fill_res_qp_entry = hns_roce_fill_res_qp_entry,
 };
 
 static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c
index 3f9c2f9dfdf6..e8fef37f810d 100644
--- a/drivers/infiniband/hw/hns/hns_roce_restrack.c
+++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c
@@ -78,3 +78,37 @@ int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq)
 
 	return ret;
 }
+
+int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp)
+{
+	struct hns_roce_qp *hr_qp = to_hr_qp(ib_qp);
+	struct nlattr *table_attr;
+
+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
+	if (!table_attr)
+		return -EMSGSIZE;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "sq_wqe_cnt", hr_qp->sq.wqe_cnt))
+		goto err;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "sq_max_gs", hr_qp->sq.max_gs))
+		goto err;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "rq_wqe_cnt", hr_qp->rq.wqe_cnt))
+		goto err;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "rq_max_gs", hr_qp->rq.max_gs))
+		goto err;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "ext_sge_sge_cnt", hr_qp->sge.sge_cnt))
+		goto err;
+
+	nla_nest_end(msg, table_attr);
+
+	return 0;
+
+err:
+	nla_nest_cancel(msg, table_attr);
+
+	return -EMSGSIZE;
+}
-- 
2.30.0

[PATCH v2 for-next 5/7] RDMA/hns: Support QP's restrack raw ops for hns driver

[Wenpeng Liang]

The QP raw restrack attributes come from the queue context maintained by
the ROCEE.

For example:

$ rdma res show qp link hns_0 -jp -dd -r
[ {
        "ifindex": 4,
        "ifname": "hns_0",
        "data": [ 2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,
		  5,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,255,156,0,0,63,156,0,0,
		  7,0,0,0,1,0,0,0,9,0,0,0,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,
		  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
		  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,63,156,0,
		  0,0,0,0,0 ]
    } ]

Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
---
 drivers/infiniband/hw/hns/hns_roce_device.h   |  2 +
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c    | 12 ++--
 drivers/infiniband/hw/hns/hns_roce_main.c     |  1 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 56 +++++++++++++++++++
 4 files changed, 65 insertions(+), 6 deletions(-)

diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index 7578c0c6313b..e0395870b819 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -895,6 +895,7 @@ struct hns_roce_hw {
 	void (*cleanup_eq)(struct hns_roce_dev *hr_dev);
 	int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);
 	int (*query_cqc)(struct hns_roce_dev *hr_dev, u32 cqn, void *buffer);
+	int (*query_qpc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer);
 	const struct ib_device_ops *hns_roce_dev_ops;
 	const struct ib_device_ops *hns_roce_dev_srq_ops;
 };
@@ -1226,6 +1227,7 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev);
 int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);
 int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);
 int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp);
+int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index 979cd57a72fb..319de9a4d2ef 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -5307,9 +5307,8 @@ static int to_ib_qp_st(enum hns_roce_v2_qp_state state)
 	return (state < ARRAY_SIZE(map)) ? map[state] : -1;
 }
 
-static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
-				 struct hns_roce_qp *hr_qp,
-				 struct hns_roce_v2_qp_context *hr_context)
+static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, u32 qpn,
+				 void *buffer)
 {
 	struct hns_roce_cmd_mailbox *mailbox;
 	int ret;
@@ -5319,11 +5318,11 @@ static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
 		return PTR_ERR(mailbox);
 
 	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_QPC,
-				hr_qp->qpn);
+				qpn);
 	if (ret)
 		goto out;
 
-	memcpy(hr_context, mailbox->buf, hr_dev->caps.qpc_sz);
+	memcpy(buffer, mailbox->buf, hr_dev->caps.qpc_sz);
 
 out:
 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
@@ -5353,7 +5352,7 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 		goto done;
 	}
 
-	ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context);
+	ret = hns_roce_v2_query_qpc(hr_dev, hr_qp->qpn, &context);
 	if (ret) {
 		ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret);
 		ret = -EINVAL;
@@ -6645,6 +6644,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {
 	.cleanup_eq = hns_roce_v2_cleanup_eq_table,
 	.write_srqc = hns_roce_v2_write_srqc,
 	.query_cqc = hns_roce_v2_query_cqc,
+	.query_qpc = hns_roce_v2_query_qpc,
 	.hns_roce_dev_ops = &hns_roce_v2_dev_ops,
 	.hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
 };
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index 87442027b808..17bc73c108f2 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -569,6 +569,7 @@ static const struct ib_device_ops hns_roce_dev_restrack_ops = {
 	.fill_res_cq_entry = hns_roce_fill_res_cq_entry,
 	.fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw,
 	.fill_res_qp_entry = hns_roce_fill_res_qp_entry,
+	.fill_res_qp_entry_raw = hns_roce_fill_res_qp_entry_raw,
 };
 
 static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c
index e8fef37f810d..9bafc627864b 100644
--- a/drivers/infiniband/hw/hns/hns_roce_restrack.c
+++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c
@@ -112,3 +112,59 @@ int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp)
 
 	return -EMSGSIZE;
 }
+
+int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_qp->device);
+	struct hns_roce_qp *hr_qp = to_hr_qp(ib_qp);
+	struct hns_roce_v2_qp_context context;
+	u32 data[MAX_ENTRY_NUM] = {};
+	int offset = 0;
+	int ret;
+
+	if (!hr_dev->hw->query_qpc)
+		return -EINVAL;
+
+	ret = hr_dev->hw->query_qpc(hr_dev, hr_qp->qpn, &context);
+	if (ret)
+		return -EINVAL;
+
+	data[offset++] = hr_reg_read(&context, QPC_QP_ST);
+	data[offset++] = hr_reg_read(&context, QPC_ERR_TYPE);
+	data[offset++] = hr_reg_read(&context, QPC_CHECK_FLG);
+	data[offset++] = hr_reg_read(&context, QPC_SRQ_EN);
+	data[offset++] = hr_reg_read(&context, QPC_SRQN);
+	data[offset++] = hr_reg_read(&context, QPC_QKEY_XRCD);
+	data[offset++] = hr_reg_read(&context, QPC_TX_CQN);
+	data[offset++] = hr_reg_read(&context, QPC_RX_CQN);
+	data[offset++] = hr_reg_read(&context, QPC_SQ_PRODUCER_IDX);
+	data[offset++] = hr_reg_read(&context, QPC_SQ_CONSUMER_IDX);
+	data[offset++] = hr_reg_read(&context, QPC_RQ_RECORD_EN);
+	data[offset++] = hr_reg_read(&context, QPC_RQ_PRODUCER_IDX);
+	data[offset++] = hr_reg_read(&context, QPC_RQ_CONSUMER_IDX);
+	data[offset++] = hr_reg_read(&context, QPC_SQ_SHIFT);
+	data[offset++] = hr_reg_read(&context, QPC_RQWS);
+	data[offset++] = hr_reg_read(&context, QPC_RQ_SHIFT);
+	data[offset++] = hr_reg_read(&context, QPC_SGE_SHIFT);
+	data[offset++] = hr_reg_read(&context, QPC_SQ_HOP_NUM);
+	data[offset++] = hr_reg_read(&context, QPC_RQ_HOP_NUM);
+	data[offset++] = hr_reg_read(&context, QPC_SGE_HOP_NUM);
+	data[offset++] = hr_reg_read(&context, QPC_WQE_SGE_BA_PG_SZ);
+	data[offset++] = hr_reg_read(&context, QPC_WQE_SGE_BUF_PG_SZ);
+	data[offset++] = hr_reg_read(&context, QPC_RETRY_NUM_INIT);
+	data[offset++] = hr_reg_read(&context, QPC_RETRY_CNT);
+	data[offset++] = hr_reg_read(&context, QPC_SQ_CUR_PSN);
+	data[offset++] = hr_reg_read(&context, QPC_SQ_MAX_PSN);
+	data[offset++] = hr_reg_read(&context, QPC_SQ_FLUSH_IDX);
+	data[offset++] = hr_reg_read(&context, QPC_SQ_MAX_IDX);
+	data[offset++] = hr_reg_read(&context, QPC_SQ_TX_ERR);
+	data[offset++] = hr_reg_read(&context, QPC_SQ_RX_ERR);
+	data[offset++] = hr_reg_read(&context, QPC_RQ_RX_ERR);
+	data[offset++] = hr_reg_read(&context, QPC_RQ_TX_ERR);
+	data[offset++] = hr_reg_read(&context, QPC_RQ_CQE_IDX);
+	data[offset++] = hr_reg_read(&context, QPC_RQ_RTY_TX_ERR);
+
+	ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, offset * sizeof(u32), data);
+
+	return ret;
+}
-- 
2.30.0

[PATCH v2 for-next 6/7] RDMA/hns: Support MR's restrack ops for hns driver

[Wenpeng Liang]

The MR restrack attributes come from the queue information maintained by
the driver.

For example:

$ rdma res show mr dev hns_0 mrn 6 -dd -jp
[ {
        "ifindex": 4,
        "ifname": "hns_0",
        "mrn": 6,
        "rkey": "300",
        "lkey": "300",
        "mrlen": 131072,
        "pdn": 8,
        "pid": 1524,
        "comm": "ib_send_bw"
    },
    "drv_pbl_hop_num": 2,
    "drv_ba_pg_shift": 14,
    "drv_buf_pg_shift": 12
}

Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
---
 drivers/infiniband/hw/hns/hns_roce_device.h   |  1 +
 drivers/infiniband/hw/hns/hns_roce_main.c     |  1 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 30 +++++++++++++++++++
 3 files changed, 32 insertions(+)

diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index e0395870b819..30a67bc70f1a 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -1228,6 +1228,7 @@ int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);
 int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);
 int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp);
 int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp);
+int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index 17bc73c108f2..ff4386b5c064 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -570,6 +570,7 @@ static const struct ib_device_ops hns_roce_dev_restrack_ops = {
 	.fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw,
 	.fill_res_qp_entry = hns_roce_fill_res_qp_entry,
 	.fill_res_qp_entry_raw = hns_roce_fill_res_qp_entry_raw,
+	.fill_res_mr_entry = hns_roce_fill_res_mr_entry,
 };
 
 static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c
index 9bafc627864b..84f942e19743 100644
--- a/drivers/infiniband/hw/hns/hns_roce_restrack.c
+++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c
@@ -168,3 +168,33 @@ int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp)
 
 	return ret;
 }
+
+int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr)
+{
+	struct hns_roce_mr *hr_mr = to_hr_mr(ib_mr);
+	struct nlattr *table_attr;
+
+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
+	if (!table_attr)
+		return -EMSGSIZE;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "pbl_hop_num", hr_mr->pbl_hop_num))
+		goto err;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "ba_pg_shift",
+				       hr_mr->pbl_mtr.hem_cfg.ba_pg_shift))
+		goto err;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "buf_pg_shift",
+				       hr_mr->pbl_mtr.hem_cfg.buf_pg_shift))
+		goto err;
+
+	nla_nest_end(msg, table_attr);
+
+	return 0;
+
+err:
+	nla_nest_cancel(msg, table_attr);
+
+	return -EMSGSIZE;
+}
-- 
2.30.0

[PATCH v2 for-next 7/7] RDMA/hns: Support MR's restrack raw ops for hns driver

[Wenpeng Liang]

The MR raw restrack attributes come from the queue context maintained by
the ROCEE.

For example:

$ rdma res show mr dev hns_0 mrn 6 -dd -jp -r
[ {
        "ifindex": 4,
        "ifname": "hns_0",
        "data": [ 1,0,0,0,2,0,0,0,0,3,0,0,0,0,2,0,0,0,0,0,32,0,0,0,2,0,0,0,
		  2,0,0,0,0,0,0,0 ]
    } ]

Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
---
 drivers/infiniband/hw/hns/hns_roce_device.h   |  2 ++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c    | 29 +++++++++++++++++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h    |  3 +-
 drivers/infiniband/hw/hns/hns_roce_main.c     |  1 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 31 +++++++++++++++++++
 5 files changed, 65 insertions(+), 1 deletion(-)

diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index 30a67bc70f1a..1bcecc5589fa 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -896,6 +896,7 @@ struct hns_roce_hw {
 	int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);
 	int (*query_cqc)(struct hns_roce_dev *hr_dev, u32 cqn, void *buffer);
 	int (*query_qpc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer);
+	int (*query_mpt)(struct hns_roce_dev *hr_dev, u32 key, void *buffer);
 	const struct ib_device_ops *hns_roce_dev_ops;
 	const struct ib_device_ops *hns_roce_dev_srq_ops;
 };
@@ -1229,6 +1230,7 @@ int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);
 int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp);
 int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp);
 int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr);
+int hns_roce_fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ib_mr);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index 319de9a4d2ef..f59176561420 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -5802,6 +5802,34 @@ static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn,
 	return ret;
 }
 
+int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key, void *buffer)
+{
+	struct hns_roce_v2_mpt_entry *context;
+	struct hns_roce_cmd_mailbox *mailbox;
+	int ret;
+
+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+	if (IS_ERR(mailbox))
+		return PTR_ERR(mailbox);
+
+	context = mailbox->buf;
+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT,
+				key_to_hw_index(key));
+	if (ret) {
+		ibdev_err(&hr_dev->ib_dev,
+			  "failed to process cmd when querying MPT, ret = %d.\n",
+			  ret);
+		goto err_mailbox;
+	}
+
+	memcpy(buffer, context, sizeof(*context));
+
+err_mailbox:
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+
+	return ret;
+}
+
 static void hns_roce_irq_work_handle(struct work_struct *work)
 {
 	struct hns_roce_work *irq_work =
@@ -6645,6 +6673,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {
 	.write_srqc = hns_roce_v2_write_srqc,
 	.query_cqc = hns_roce_v2_query_cqc,
 	.query_qpc = hns_roce_v2_query_qpc,
+	.query_mpt = hns_roce_v2_query_mpt,
 	.hns_roce_dev_ops = &hns_roce_v2_dev_ops,
 	.hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
 };
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
index 49ec29973ed7..ae29780dd63a 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
@@ -758,7 +758,8 @@ struct hns_roce_v2_mpt_entry {
 #define MPT_INNER_PA_VLD MPT_FIELD_LOC(71, 71)
 #define MPT_MW_BIND_QPN MPT_FIELD_LOC(95, 72)
 #define MPT_BOUND_LKEY MPT_FIELD_LOC(127, 96)
-#define MPT_LEN MPT_FIELD_LOC(191, 128)
+#define MPT_LEN_L MPT_FIELD_LOC(159, 128)
+#define MPT_LEN_H MPT_FIELD_LOC(191, 160)
 #define MPT_LKEY MPT_FIELD_LOC(223, 192)
 #define MPT_VA MPT_FIELD_LOC(287, 224)
 #define MPT_PBL_SIZE MPT_FIELD_LOC(319, 288)
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index ff4386b5c064..9de3a522980a 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -571,6 +571,7 @@ static const struct ib_device_ops hns_roce_dev_restrack_ops = {
 	.fill_res_qp_entry = hns_roce_fill_res_qp_entry,
 	.fill_res_qp_entry_raw = hns_roce_fill_res_qp_entry_raw,
 	.fill_res_mr_entry = hns_roce_fill_res_mr_entry,
+	.fill_res_mr_entry_raw = hns_roce_fill_res_mr_entry_raw,
 };
 
 static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c
index 84f942e19743..989a2af2e938 100644
--- a/drivers/infiniband/hw/hns/hns_roce_restrack.c
+++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c
@@ -198,3 +198,34 @@ int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr)
 
 	return -EMSGSIZE;
 }
+
+int hns_roce_fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ib_mr)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_mr->device);
+	struct hns_roce_mr *hr_mr = to_hr_mr(ib_mr);
+	struct hns_roce_v2_mpt_entry context;
+	u32 data[MAX_ENTRY_NUM] = {};
+	int offset = 0;
+	int ret;
+
+	if (!hr_dev->hw->query_mpt)
+		return -EINVAL;
+
+	ret = hr_dev->hw->query_mpt(hr_dev, hr_mr->key, &context);
+	if (ret)
+		return -EINVAL;
+
+	data[offset++] = hr_reg_read(&context, MPT_ST);
+	data[offset++] = hr_reg_read(&context, MPT_PD);
+	data[offset++] = hr_reg_read(&context, MPT_LKEY);
+	data[offset++] = hr_reg_read(&context, MPT_LEN_L);
+	data[offset++] = hr_reg_read(&context, MPT_LEN_H);
+	data[offset++] = hr_reg_read(&context, MPT_PBL_SIZE);
+	data[offset++] = hr_reg_read(&context, MPT_PBL_HOP_NUM);
+	data[offset++] = hr_reg_read(&context, MPT_PBL_BA_PG_SZ);
+	data[offset++] = hr_reg_read(&context, MPT_PBL_BUF_PG_SZ);
+
+	ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, offset * sizeof(u32), data);
+
+	return ret;
+}
-- 
2.30.0

Re: [PATCH v2 for-next 0/7] RDMA/hns: Add more restrack attributes to hns driver

[Leon Romanovsky]

On Mon, Aug 22, 2022 at 06:44:48PM +0800, Wenpeng Liang wrote:
> The hns driver already supports CQ's restrack ops. Therefore, this patchset
> supports QP/MR's restrack ops and CQ/QP/MR's restrack raw ops. Among them,
> restrack ops dump the queue information maintained by the driver, and
> restrack raw ops dump the queue context maintained by ROCEE.
> 
> Changes since v1:
> * Add support for restrack raw ops.
> * Add the result dumped by rdmatool in the commit message.
> * v1 Link: https://patchwork.kernel.org/project/linux-rdma/cover/20220124124624.55352-1-liangwenpeng@huawei.com/

https://lore.kernel.org/all/20220124124624.55352-1-liangwenpeng@huawei.com/

> 
> Wenpeng Liang (7):
>   RDMA/hns: Remove redundant DFX file and DFX ops structure
>   RDMA/hns: Add or remove CQ's restrack attributes
>   RDMA/hns: Support CQ's restrack raw ops for hns driver
>   RDMA/hns: Support QP's restrack ops for hns driver
>   RDMA/hns: Support QP's restrack raw ops for hns driver
>   RDMA/hns: Support MR's restrack ops for hns driver
>   RDMA/hns: Support MR's restrack raw ops for hns driver
> 

Thanks, applied.