* [PATCH for-next 1/2] RDMA/rtrs-clt: Add event tracing support
2022-08-18 10:52 [PATCH for-next 0/2] Add trace event support to RTRS Md Haris Iqbal
@ 2022-08-18 10:52 ` Md Haris Iqbal
2022-08-18 10:52 ` [PATCH for-next 2/2] RDMA/rtrs-srv: " Md Haris Iqbal
2022-08-21 9:09 ` [PATCH for-next 0/2] Add trace event support to RTRS Leon Romanovsky
2 siblings, 0 replies; 4+ messages in thread
From: Md Haris Iqbal @ 2022-08-18 10:52 UTC (permalink / raw)
To: linux-rdma; +Cc: leon, jgg, haris.iqbal, jinpu.wang, Santosh Pradhan
From: Santosh Pradhan <santosh.pradhan@ionos.com>
Add event tracing mechanism for following routines:
- rtrs_clt_reconnect_work()
- rtrs_clt_close_conns()
- rtrs_rdma_error_recovery()
How to use:
1. Load the rtrs_client module
2. cd /sys/kernel/debug/tracing
3. If all the events need to be enabled:
echo 1 > events/rtrs_clt/enable
4. OR only speific routine/event needs to be enabled e.g.
echo 1 > events/rtrs_clt/rtrs_clt_close_conns/enable
5. cat trace
6. Run some workload which can trigger rtrs_clt_close_conns()
Signed-off-by: Santosh Pradhan <santosh.pradhan@ionos.com>
Signed-off-by: Jack Wang <jinpu.wang@ionos.com>
Signed-off-by: Md Haris Iqbal <haris.iqbal@ionos.com>
---
drivers/infiniband/ulp/rtrs/Makefile | 5 +-
drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c | 15 ++++
drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h | 86 ++++++++++++++++++++
drivers/infiniband/ulp/rtrs/rtrs-clt.c | 7 ++
4 files changed, 112 insertions(+), 1 deletion(-)
create mode 100644 drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c
create mode 100644 drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h
diff --git a/drivers/infiniband/ulp/rtrs/Makefile b/drivers/infiniband/ulp/rtrs/Makefile
index 3898509be270..1fdf918b37eb 100644
--- a/drivers/infiniband/ulp/rtrs/Makefile
+++ b/drivers/infiniband/ulp/rtrs/Makefile
@@ -1,8 +1,11 @@
# SPDX-License-Identifier: GPL-2.0-or-later
+CFLAGS_rtrs-clt-trace.o = -I$(src)
+
rtrs-client-y := rtrs-clt.o \
rtrs-clt-stats.o \
- rtrs-clt-sysfs.o
+ rtrs-clt-sysfs.o \
+ rtrs-clt-trace.o
rtrs-server-y := rtrs-srv.o \
rtrs-srv-stats.o \
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c
new file mode 100644
index 000000000000..f14fa1f36ce8
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2022 1&1 IONOS SE. All rights reserved.
+ */
+#include "rtrs.h"
+#include "rtrs-clt.h"
+
+/*
+ * We include this last to have the helpers above available for the trace
+ * event implementations.
+ */
+#define CREATE_TRACE_POINTS
+#include "rtrs-clt-trace.h"
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h
new file mode 100644
index 000000000000..7738e2676855
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h
@@ -0,0 +1,86 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2022 1&1 IONOS SE. All rights reserved.
+ */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM rtrs_clt
+
+#if !defined(_TRACE_RTRS_CLT_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_RTRS_CLT_H
+
+#include <linux/tracepoint.h>
+
+struct rtrs_clt_path;
+struct rtrs_clt_sess;
+
+TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTING);
+TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTING_ERR);
+TRACE_DEFINE_ENUM(RTRS_CLT_RECONNECTING);
+TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTED);
+TRACE_DEFINE_ENUM(RTRS_CLT_CLOSING);
+TRACE_DEFINE_ENUM(RTRS_CLT_CLOSED);
+TRACE_DEFINE_ENUM(RTRS_CLT_DEAD);
+
+#define show_rtrs_clt_state(x) \
+ __print_symbolic(x, \
+ { RTRS_CLT_CONNECTING, "CONNECTING" }, \
+ { RTRS_CLT_CONNECTING_ERR, "CONNECTING_ERR" }, \
+ { RTRS_CLT_RECONNECTING, "RECONNECTING" }, \
+ { RTRS_CLT_CONNECTED, "CONNECTED" }, \
+ { RTRS_CLT_CLOSING, "CLOSING" }, \
+ { RTRS_CLT_CLOSED, "CLOSED" }, \
+ { RTRS_CLT_DEAD, "DEAD" })
+
+DECLARE_EVENT_CLASS(rtrs_clt_conn_class,
+ TP_PROTO(struct rtrs_clt_path *clt_path),
+
+ TP_ARGS(clt_path),
+
+ TP_STRUCT__entry(
+ __field(int, state)
+ __field(int, reconnect_attempts)
+ __field(int, max_reconnect_attempts)
+ __field(int, fail_cnt)
+ __field(int, success_cnt)
+ __array(char, sessname, NAME_MAX)
+ ),
+
+ TP_fast_assign(
+ struct rtrs_clt_sess *clt = clt_path->clt;
+
+ __entry->state = clt_path->state;
+ __entry->reconnect_attempts = clt_path->reconnect_attempts;
+ __entry->max_reconnect_attempts = clt->max_reconnect_attempts;
+ __entry->fail_cnt = clt_path->stats->reconnects.fail_cnt;
+ __entry->success_cnt = clt_path->stats->reconnects.successful_cnt;
+ memcpy(__entry->sessname, kobject_name(&clt_path->kobj), NAME_MAX);
+ ),
+
+ TP_printk("RTRS-CLT: sess='%s' state=%s attempts='%d' max-attempts='%d' fail='%d' success='%d'",
+ __entry->sessname,
+ show_rtrs_clt_state(__entry->state),
+ __entry->reconnect_attempts,
+ __entry->max_reconnect_attempts,
+ __entry->fail_cnt,
+ __entry->success_cnt
+ )
+);
+
+#define DEFINE_CLT_CONN_EVENT(name) \
+DEFINE_EVENT(rtrs_clt_conn_class, rtrs_##name, \
+ TP_PROTO(struct rtrs_clt_path *clt_path), \
+ TP_ARGS(clt_path))
+
+DEFINE_CLT_CONN_EVENT(clt_reconnect_work);
+DEFINE_CLT_CONN_EVENT(clt_close_conns);
+DEFINE_CLT_CONN_EVENT(rdma_error_recovery);
+
+#endif /* _TRACE_RTRS_CLT_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE rtrs-clt-trace
+#include <trace/define_trace.h>
+
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
index baecde41d126..5219bb10777a 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
@@ -16,6 +16,7 @@
#include "rtrs-clt.h"
#include "rtrs-log.h"
+#include "rtrs-clt-trace.h"
#define RTRS_CONNECT_TIMEOUT_MS 30000
/*
@@ -302,6 +303,8 @@ static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con)
{
struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
+ trace_rtrs_rdma_error_recovery(clt_path);
+
if (rtrs_clt_change_state_from_to(clt_path,
RTRS_CLT_CONNECTED,
RTRS_CLT_RECONNECTING)) {
@@ -1942,6 +1945,8 @@ static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con,
void rtrs_clt_close_conns(struct rtrs_clt_path *clt_path, bool wait)
{
+ trace_rtrs_clt_close_conns(clt_path);
+
if (rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CLOSING, NULL))
queue_work(rtrs_wq, &clt_path->close_work);
if (wait)
@@ -2648,6 +2653,8 @@ static void rtrs_clt_reconnect_work(struct work_struct *work)
reconnect_dwork);
clt = clt_path->clt;
+ trace_rtrs_clt_reconnect_work(clt_path);
+
if (READ_ONCE(clt_path->state) != RTRS_CLT_RECONNECTING)
return;
--
2.25.1
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH for-next 2/2] RDMA/rtrs-srv: Add event tracing support
2022-08-18 10:52 [PATCH for-next 0/2] Add trace event support to RTRS Md Haris Iqbal
2022-08-18 10:52 ` [PATCH for-next 1/2] RDMA/rtrs-clt: Add event tracing support Md Haris Iqbal
@ 2022-08-18 10:52 ` Md Haris Iqbal
2022-08-21 9:09 ` [PATCH for-next 0/2] Add trace event support to RTRS Leon Romanovsky
2 siblings, 0 replies; 4+ messages in thread
From: Md Haris Iqbal @ 2022-08-18 10:52 UTC (permalink / raw)
To: linux-rdma; +Cc: leon, jgg, haris.iqbal, jinpu.wang, Santosh Pradhan
From: Santosh Pradhan <santosh.pradhan@ionos.com>
Add event tracing mechanism for following routines:
- send_io_resp_imm()
How to use:
1. Load the rtrs_server module
2. cd /sys/kernel/debug/tracing
3. If all the events need to be enabled:
echo 1 > events/rtrs_srv/enable
4. OR only speific routine/event needs to be enabled e.g.
echo 1 > events/rtrs_srv/send_io_resp_imm/enable
5. cat trace
6. Run some I/O workload which can trigger send_io_resp_imm()
Signed-off-by: Santosh Pradhan <santosh.pradhan@ionos.com>
Signed-off-by: Jack Wang <jinpu.wang@ionos.com>
Signed-off-by: Md Haris Iqbal <haris.iqbal@ionos.com>
---
drivers/infiniband/ulp/rtrs/Makefile | 5 +-
drivers/infiniband/ulp/rtrs/rtrs-srv-trace.c | 16 ++++
drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h | 88 ++++++++++++++++++++
drivers/infiniband/ulp/rtrs/rtrs-srv.c | 8 +-
drivers/infiniband/ulp/rtrs/rtrs-srv.h | 5 ++
5 files changed, 116 insertions(+), 6 deletions(-)
create mode 100644 drivers/infiniband/ulp/rtrs/rtrs-srv-trace.c
create mode 100644 drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h
diff --git a/drivers/infiniband/ulp/rtrs/Makefile b/drivers/infiniband/ulp/rtrs/Makefile
index 1fdf918b37eb..5227e7788e1f 100644
--- a/drivers/infiniband/ulp/rtrs/Makefile
+++ b/drivers/infiniband/ulp/rtrs/Makefile
@@ -7,9 +7,12 @@ rtrs-client-y := rtrs-clt.o \
rtrs-clt-sysfs.o \
rtrs-clt-trace.o
+CFLAGS_rtrs-srv-trace.o = -I$(src)
+
rtrs-server-y := rtrs-srv.o \
rtrs-srv-stats.o \
- rtrs-srv-sysfs.o
+ rtrs-srv-sysfs.o \
+ rtrs-srv-trace.o
rtrs-core-y := rtrs.o
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.c b/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.c
new file mode 100644
index 000000000000..29ca59ceb0dd
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.c
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2022 1&1 IONOS SE. All rights reserved.
+ */
+#include "rtrs.h"
+#include "rtrs-pri.h"
+#include "rtrs-srv.h"
+
+/*
+ * We include this last to have the helpers above available for the trace
+ * event implementations.
+ */
+#define CREATE_TRACE_POINTS
+#include "rtrs-srv-trace.h"
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h b/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h
new file mode 100644
index 000000000000..587d3e033081
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h
@@ -0,0 +1,88 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2022 1&1 IONOS SE. All rights reserved.
+ */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM rtrs_srv
+
+#if !defined(_TRACE_RTRS_SRV_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_RTRS_SRV_H
+
+#include <linux/tracepoint.h>
+
+struct rtrs_srv_op;
+struct rtrs_srv_con;
+struct rtrs_srv_path;
+
+TRACE_DEFINE_ENUM(RTRS_SRV_CONNECTING);
+TRACE_DEFINE_ENUM(RTRS_SRV_CONNECTED);
+TRACE_DEFINE_ENUM(RTRS_SRV_CLOSING);
+TRACE_DEFINE_ENUM(RTRS_SRV_CLOSED);
+
+#define show_rtrs_srv_state(x) \
+ __print_symbolic(x, \
+ { RTRS_SRV_CONNECTING, "CONNECTING" }, \
+ { RTRS_SRV_CONNECTED, "CONNECTED" }, \
+ { RTRS_SRV_CLOSING, "CLOSING" }, \
+ { RTRS_SRV_CLOSED, "CLOSED" })
+
+TRACE_EVENT(send_io_resp_imm,
+ TP_PROTO(struct rtrs_srv_op *id,
+ bool need_inval,
+ bool always_invalidate,
+ int errno),
+
+ TP_ARGS(id, need_inval, always_invalidate, errno),
+
+ TP_STRUCT__entry(
+ __field(u8, dir)
+ __field(bool, need_inval)
+ __field(bool, always_invalidate)
+ __field(u32, msg_id)
+ __field(int, wr_cnt)
+ __field(u32, signal_interval)
+ __field(int, state)
+ __field(int, errno)
+ __array(char, sessname, NAME_MAX)
+ ),
+
+ TP_fast_assign(
+ struct rtrs_srv_con *con = id->con;
+ struct rtrs_path *s = con->c.path;
+ struct rtrs_srv_path *srv_path = to_srv_path(s);
+
+ __entry->dir = id->dir;
+ __entry->state = srv_path->state;
+ __entry->errno = errno;
+ __entry->need_inval = need_inval;
+ __entry->always_invalidate = always_invalidate;
+ __entry->msg_id = id->msg_id;
+ __entry->wr_cnt = atomic_read(&con->c.wr_cnt);
+ __entry->signal_interval = s->signal_interval;
+ memcpy(__entry->sessname, kobject_name(&srv_path->kobj), NAME_MAX);
+ ),
+
+ TP_printk("sess='%s' state='%s' dir=%s err='%d' inval='%d' glob-inval='%d' msgid='%u' wrcnt='%d' sig-interval='%u'",
+ __entry->sessname,
+ show_rtrs_srv_state(__entry->state),
+ __print_symbolic(__entry->dir,
+ { READ, "READ" },
+ { WRITE, "WRITE" }),
+ __entry->errno,
+ __entry->need_inval,
+ __entry->always_invalidate,
+ __entry->msg_id,
+ __entry->wr_cnt,
+ __entry->signal_interval
+ )
+);
+
+#endif /* _TRACE_RTRS_SRV_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE rtrs-srv-trace
+#include <trace/define_trace.h>
+
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
index 34c03bde5064..22e6f991946c 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
@@ -16,6 +16,7 @@
#include "rtrs-log.h"
#include <rdma/ib_cm.h>
#include <rdma/ib_verbs.h>
+#include "rtrs-srv-trace.h"
MODULE_DESCRIPTION("RDMA Transport Server");
MODULE_LICENSE("GPL");
@@ -57,11 +58,6 @@ static inline struct rtrs_srv_con *to_srv_con(struct rtrs_con *c)
return container_of(c, struct rtrs_srv_con, c);
}
-static inline struct rtrs_srv_path *to_srv_path(struct rtrs_path *s)
-{
- return container_of(s, struct rtrs_srv_path, s);
-}
-
static bool rtrs_srv_change_state(struct rtrs_srv_path *srv_path,
enum rtrs_srv_state new_state)
{
@@ -375,6 +371,8 @@ static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id,
}
}
+ trace_send_io_resp_imm(id, need_inval, always_invalidate, errno);
+
if (need_inval && always_invalidate) {
wr = &inv_wr;
inv_wr.next = &rwr.wr;
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.h b/drivers/infiniband/ulp/rtrs/rtrs-srv.h
index 186a63c217df..2f8a638e36fa 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-srv.h
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.h
@@ -91,6 +91,11 @@ struct rtrs_srv_path {
struct rtrs_srv_stats *stats;
};
+static inline struct rtrs_srv_path *to_srv_path(struct rtrs_path *s)
+{
+ return container_of(s, struct rtrs_srv_path, s);
+}
+
struct rtrs_srv_sess {
struct list_head paths_list;
int paths_up;
--
2.25.1
^ permalink raw reply related [flat|nested] 4+ messages in thread* Re: [PATCH for-next 0/2] Add trace event support to RTRS
2022-08-18 10:52 [PATCH for-next 0/2] Add trace event support to RTRS Md Haris Iqbal
2022-08-18 10:52 ` [PATCH for-next 1/2] RDMA/rtrs-clt: Add event tracing support Md Haris Iqbal
2022-08-18 10:52 ` [PATCH for-next 2/2] RDMA/rtrs-srv: " Md Haris Iqbal
@ 2022-08-21 9:09 ` Leon Romanovsky
2 siblings, 0 replies; 4+ messages in thread
From: Leon Romanovsky @ 2022-08-21 9:09 UTC (permalink / raw)
To: Md Haris Iqbal; +Cc: linux-rdma, jgg, jinpu.wang
On Thu, Aug 18, 2022 at 12:52:38PM +0200, Md Haris Iqbal wrote:
> Hi Jason, hi Leon,
>
> Please consider to include following changes to the next merge window.
>
> The patchset is organized as follows:
> 1: Adds trace event to rtrs-clt
> 2: Adds trace event to rtrs-srv
>
> Santosh Pradhan (2):
> RDMA/rtrs-clt: Add event tracing support
> RDMA/rtrs-srv: Add event tracing support
>
Thanks, applied.
^ permalink raw reply [flat|nested] 4+ messages in thread