From: Chuck Lever <chuck.lever@oracle.com>
To: dledford@redhat.com, jgg@mellanox.com, leon@kernel.org
Cc: linux-rdma@vger.kernel.org
Subject: [PATCH v10 3/3] RDMA/core: Add trace points to follow MR allocation
Date: Wed, 18 Dec 2019 15:18:20 -0500 [thread overview]
Message-ID: <20191218201820.30584.34636.stgit@manet.1015granger.net> (raw)
In-Reply-To: <20191218201631.30584.53987.stgit@manet.1015granger.net>
Track the lifetime of ib_mr objects. Here's sample output from a
test run with NFS/RDMA:
<...>-361 [009] 79238.772782: mr_alloc: pd.id=3 mr.id=11 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.772812: mr_alloc: pd.id=3 mr.id=12 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.772839: mr_alloc: pd.id=3 mr.id=13 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.772866: mr_alloc: pd.id=3 mr.id=14 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.772893: mr_alloc: pd.id=3 mr.id=15 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.772921: mr_alloc: pd.id=3 mr.id=16 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.772947: mr_alloc: pd.id=3 mr.id=17 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.772974: mr_alloc: pd.id=3 mr.id=18 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.773001: mr_alloc: pd.id=3 mr.id=19 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.773028: mr_alloc: pd.id=3 mr.id=20 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.773055: mr_alloc: pd.id=3 mr.id=21 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.270942: mr_alloc: pd.id=3 mr.id=22 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.270975: mr_alloc: pd.id=3 mr.id=23 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271007: mr_alloc: pd.id=3 mr.id=24 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271036: mr_alloc: pd.id=3 mr.id=25 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271067: mr_alloc: pd.id=3 mr.id=26 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271095: mr_alloc: pd.id=3 mr.id=27 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271121: mr_alloc: pd.id=3 mr.id=28 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271153: mr_alloc: pd.id=3 mr.id=29 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271181: mr_alloc: pd.id=3 mr.id=30 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271208: mr_alloc: pd.id=3 mr.id=31 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271236: mr_alloc: pd.id=3 mr.id=32 type=MEM_REG max_num_sg=30 rc=0
<...>-4351 [001] 79242.299400: mr_dereg: mr.id=32
<...>-4351 [001] 79242.299467: mr_dereg: mr.id=31
<...>-4351 [001] 79242.299554: mr_dereg: mr.id=30
<...>-4351 [001] 79242.299615: mr_dereg: mr.id=29
<...>-4351 [001] 79242.299684: mr_dereg: mr.id=28
<...>-4351 [001] 79242.299748: mr_dereg: mr.id=27
<...>-4351 [001] 79242.299812: mr_dereg: mr.id=26
<...>-4351 [001] 79242.299874: mr_dereg: mr.id=25
<...>-4351 [001] 79242.299944: mr_dereg: mr.id=24
<...>-4351 [001] 79242.300009: mr_dereg: mr.id=23
<...>-4351 [001] 79242.300190: mr_dereg: mr.id=22
<...>-4351 [001] 79242.300263: mr_dereg: mr.id=21
<...>-4351 [001] 79242.300326: mr_dereg: mr.id=20
<...>-4351 [001] 79242.300388: mr_dereg: mr.id=19
<...>-4351 [001] 79242.300450: mr_dereg: mr.id=18
<...>-4351 [001] 79242.300516: mr_dereg: mr.id=17
<...>-4351 [001] 79242.300629: mr_dereg: mr.id=16
<...>-4351 [001] 79242.300718: mr_dereg: mr.id=15
<...>-4351 [001] 79242.300784: mr_dereg: mr.id=14
<...>-4351 [001] 79242.300879: mr_dereg: mr.id=13
<...>-4351 [001] 79242.300945: mr_dereg: mr.id=12
<...>-4351 [001] 79242.301012: mr_dereg: mr.id=11
Some features of the output:
- The lifetime and owner PD of each MR is clearly visible.
- The type of MR is captured, as is the SGE array size.
- Failing MR allocation can be recorded.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
drivers/infiniband/core/verbs.c | 39 +++++++++---
include/trace/events/rdma_core.h | 123 ++++++++++++++++++++++++++++++++++++++
2 files changed, 151 insertions(+), 11 deletions(-)
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 289b2f7a9d5e..47d54c31eb2a 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -52,6 +52,7 @@
#include <rdma/rw.h>
#include "core_priv.h"
+#include <trace/events/rdma_core.h>
#include <trace/events/rdma_core.h>
@@ -1999,6 +2000,7 @@ int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)
struct ib_sig_attrs *sig_attrs = mr->sig_attrs;
int ret;
+ trace_mr_dereg(mr);
rdma_restrack_del(&mr->res);
ret = mr->device->ops.dereg_mr(mr, udata);
if (!ret) {
@@ -2030,11 +2032,16 @@ struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type,
{
struct ib_mr *mr;
- if (!pd->device->ops.alloc_mr)
- return ERR_PTR(-EOPNOTSUPP);
+ if (!pd->device->ops.alloc_mr) {
+ mr = ERR_PTR(-EOPNOTSUPP);
+ goto out;
+ }
- if (WARN_ON_ONCE(mr_type == IB_MR_TYPE_INTEGRITY))
- return ERR_PTR(-EINVAL);
+ if (mr_type == IB_MR_TYPE_INTEGRITY) {
+ WARN_ON_ONCE(1);
+ mr = ERR_PTR(-EINVAL);
+ goto out;
+ }
mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg, udata);
if (!IS_ERR(mr)) {
@@ -2050,6 +2057,8 @@ struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type,
mr->sig_attrs = NULL;
}
+out:
+ trace_mr_alloc(pd, mr_type, max_num_sg, mr);
return mr;
}
EXPORT_SYMBOL(ib_alloc_mr_user);
@@ -2074,21 +2083,27 @@ struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
struct ib_sig_attrs *sig_attrs;
if (!pd->device->ops.alloc_mr_integrity ||
- !pd->device->ops.map_mr_sg_pi)
- return ERR_PTR(-EOPNOTSUPP);
+ !pd->device->ops.map_mr_sg_pi) {
+ mr = ERR_PTR(-EOPNOTSUPP);
+ goto out;
+ }
- if (!max_num_meta_sg)
- return ERR_PTR(-EINVAL);
+ if (!max_num_meta_sg) {
+ mr = ERR_PTR(-EINVAL);
+ goto out;
+ }
sig_attrs = kzalloc(sizeof(struct ib_sig_attrs), GFP_KERNEL);
- if (!sig_attrs)
- return ERR_PTR(-ENOMEM);
+ if (!sig_attrs) {
+ mr = ERR_PTR(-ENOMEM);
+ goto out;
+ }
mr = pd->device->ops.alloc_mr_integrity(pd, max_num_data_sg,
max_num_meta_sg);
if (IS_ERR(mr)) {
kfree(sig_attrs);
- return mr;
+ goto out;
}
mr->device = pd->device;
@@ -2102,6 +2117,8 @@ struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
mr->type = IB_MR_TYPE_INTEGRITY;
mr->sig_attrs = sig_attrs;
+out:
+ trace_mr_integ_alloc(pd, max_num_data_sg, max_num_meta_sg, mr);
return mr;
}
EXPORT_SYMBOL(ib_alloc_mr_integrity);
diff --git a/include/trace/events/rdma_core.h b/include/trace/events/rdma_core.h
index 08f481554e7f..17642aa54437 100644
--- a/include/trace/events/rdma_core.h
+++ b/include/trace/events/rdma_core.h
@@ -266,6 +266,129 @@
TP_printk("cq.id=%u", __entry->cq_id)
);
+/**
+ ** Memory Region events
+ **/
+
+/*
+ * enum ib_mr_type, from include/rdma/ib_verbs.h
+ */
+#define IB_MR_TYPE_LIST \
+ ib_mr_type_item(MEM_REG) \
+ ib_mr_type_item(SG_GAPS) \
+ ib_mr_type_item(DM) \
+ ib_mr_type_item(USER) \
+ ib_mr_type_item(DMA) \
+ ib_mr_type_end(INTEGRITY)
+
+#undef ib_mr_type_item
+#undef ib_mr_type_end
+
+#define ib_mr_type_item(x) TRACE_DEFINE_ENUM(IB_MR_TYPE_##x);
+#define ib_mr_type_end(x) TRACE_DEFINE_ENUM(IB_MR_TYPE_##x);
+
+IB_MR_TYPE_LIST
+
+#undef ib_mr_type_item
+#undef ib_mr_type_end
+
+#define ib_mr_type_item(x) { IB_MR_TYPE_##x, #x },
+#define ib_mr_type_end(x) { IB_MR_TYPE_##x, #x }
+
+#define rdma_show_ib_mr_type(x) \
+ __print_symbolic(x, IB_MR_TYPE_LIST)
+
+TRACE_EVENT(mr_alloc,
+ TP_PROTO(
+ const struct ib_pd *pd,
+ enum ib_mr_type mr_type,
+ u32 max_num_sg,
+ const struct ib_mr *mr
+ ),
+
+ TP_ARGS(pd, mr_type, max_num_sg, mr),
+
+ TP_STRUCT__entry(
+ __field(u32, pd_id)
+ __field(u32, mr_id)
+ __field(u32, max_num_sg)
+ __field(int, rc)
+ __field(unsigned long, mr_type)
+ ),
+
+ TP_fast_assign(
+ __entry->pd_id = pd->res.id;
+ if (IS_ERR(mr)) {
+ __entry->mr_id = 0;
+ __entry->rc = PTR_ERR(mr);
+ } else {
+ __entry->mr_id = mr->res.id;
+ __entry->rc = 0;
+ }
+ __entry->max_num_sg = max_num_sg;
+ __entry->mr_type = mr_type;
+ ),
+
+ TP_printk("pd.id=%u mr.id=%u type=%s max_num_sg=%u rc=%d",
+ __entry->pd_id, __entry->mr_id,
+ rdma_show_ib_mr_type(__entry->mr_type),
+ __entry->max_num_sg, __entry->rc)
+);
+
+TRACE_EVENT(mr_integ_alloc,
+ TP_PROTO(
+ const struct ib_pd *pd,
+ u32 max_num_data_sg,
+ u32 max_num_meta_sg,
+ const struct ib_mr *mr
+ ),
+
+ TP_ARGS(pd, max_num_data_sg, max_num_meta_sg, mr),
+
+ TP_STRUCT__entry(
+ __field(u32, pd_id)
+ __field(u32, mr_id)
+ __field(u32, max_num_data_sg)
+ __field(u32, max_num_meta_sg)
+ __field(int, rc)
+ ),
+
+ TP_fast_assign(
+ __entry->pd_id = pd->res.id;
+ if (IS_ERR(mr)) {
+ __entry->mr_id = 0;
+ __entry->rc = PTR_ERR(mr);
+ } else {
+ __entry->mr_id = mr->res.id;
+ __entry->rc = 0;
+ }
+ __entry->max_num_data_sg = max_num_data_sg;
+ __entry->max_num_meta_sg = max_num_meta_sg;
+ ),
+
+ TP_printk("pd.id=%u mr.id=%u max_num_data_sg=%u max_num_meta_sg=%u rc=%d",
+ __entry->pd_id, __entry->mr_id, __entry->max_num_data_sg,
+ __entry->max_num_meta_sg, __entry->rc)
+);
+
+TRACE_EVENT(mr_dereg,
+ TP_PROTO(
+ const struct ib_mr *mr
+ ),
+
+ TP_ARGS(mr),
+
+ TP_STRUCT__entry(
+ __field(u32, id)
+ ),
+
+ TP_fast_assign(
+ __entry->id = mr->res.id;
+ ),
+
+ TP_printk("mr.id=%u", __entry->id)
+);
+
#endif /* _TRACE_RDMA_CORE_H */
#include <trace/define_trace.h>
next prev parent reply other threads:[~2019-12-18 20:18 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-12-18 20:18 [PATCH v10 0/3] Proposed trace points for RDMA/core Chuck Lever
2019-12-18 20:18 ` [PATCH v10 1/3] RDMA/cma: Add trace points in RDMA Connection Manager Chuck Lever
2019-12-18 20:18 ` [PATCH v10 2/3] RDMA/core: Trace points for diagnosing completion queue issues Chuck Lever
2019-12-18 20:18 ` Chuck Lever [this message]
2019-12-19 13:32 ` [PATCH v10 0/3] Proposed trace points for RDMA/core Leon Romanovsky
2020-01-06 18:28 ` Chuck Lever
2020-01-07 20:23 ` Jason Gunthorpe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20191218201820.30584.34636.stgit@manet.1015granger.net \
--to=chuck.lever@oracle.com \
--cc=dledford@redhat.com \
--cc=jgg@mellanox.com \
--cc=leon@kernel.org \
--cc=linux-rdma@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox