[PATCH iproute2-rc 0/2] Control CQ adaptive moderation (RDMA DIM)

[PATCH iproute2-rc 0/2] Control CQ adaptive moderation (RDMA DIM)

[Leon Romanovsky]

From: Leon Romanovsky <leonro@mellanox.com>

Hi,

This is supplementary part of RDMA DIM feature [1] accepted
for the kernel v5.3. In this patch set Yamin extends rdmatool
to get/set as a default this adaptive-moderation setting on
IB device level and provides an information about DIM on/off
status per-CQ.

Thanks

[1] https://lore.kernel.org/linux-rdma/20190708105905.27468-1-leon@kernel.org/

Yamin Friedman (2):
  rdma: Control CQ adaptive moderation (DIM)
  rdma: Document adaptive-moderation

 man/man8/rdma-dev.8 | 16 ++++++++++++-
 rdma/dev.c          | 55 ++++++++++++++++++++++++++++++++++++++++++++-
 rdma/rdma.h         |  1 +
 rdma/res-cq.c       | 15 +++++++++++++
 rdma/utils.c        |  6 +++++
 5 files changed, 91 insertions(+), 2 deletions(-)

--
2.20.1

[PATCH iproute2-rc 1/2] rdma: Control CQ adaptive moderation (DIM)

[Leon Romanovsky]

From: Yamin Friedman <yaminf@mellanox.com>

In order to set adaptive-moderation for an ib device the command is:
rdma dev set [DEV] adaptive-moderation [on|off]

rdma dev show -d
0: mlx5_0: node_type ca fw 16.25.0319 node_guid 248a:0703:00a5:29d0
sys_image_guid 248a:0703:00a5:29d0 adaptive-moderation on
caps: <BAD_PKEY_CNTR, BAD_QKEY_CNTR, AUTO_PATH_MIG, CHANGE_PHY_PORT,
PORT_ACTIVE_EVENT, SYS_IMAGE_GUID, RC_RNR_NAK_GEN, MEM_WINDOW, XRC,
MEM_MGT_EXTENSIONS, BLOCK_MULTICAST_LOOPBACK, MEM_WINDOW_TYPE_2B,
RAW_IP_CSUM, CROSS_CHANNEL, MANAGED_FLOW_STEERING, SIGNATURE_HANDOVER,
ON_DEMAND_PAGING, SG_GAPS_REG, RAW_SCATTER_FCS, PCI_WRITE_END_PADDING>

rdma resource show cq
dev mlx5_0 cqn 0 cqe 1023 users 4 poll-ctx UNBOUND_WORKQUEUE
adaptive-moderation off comm [ib_core]

Signed-off-by: Yamin Friedman <yaminf@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
 rdma/dev.c    | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 rdma/rdma.h   |  1 +
 rdma/res-cq.c | 15 ++++++++++++++
 rdma/utils.c  |  6 ++++++
 4 files changed, 76 insertions(+), 1 deletion(-)

diff --git a/rdma/dev.c b/rdma/dev.c
index d28bf6b3..c597cba5 100644
--- a/rdma/dev.c
+++ b/rdma/dev.c
@@ -12,6 +12,7 @@ static int dev_help(struct rd *rd)
 	pr_out("Usage: %s dev show [DEV]\n", rd->filename);
 	pr_out("       %s dev set [DEV] name DEVNAME\n", rd->filename);
 	pr_out("       %s dev set [DEV] netns NSNAME\n", rd->filename);
+	pr_out("       %s dev set [DEV] adaptive-moderation [on|off]\n", rd->filename);
 	return 0;
 }
 
@@ -167,6 +168,21 @@ static void dev_print_sys_image_guid(struct rd *rd, struct nlattr **tb)
 		pr_out("sys_image_guid %s ", str);
 }
 
+static void dev_print_dim_setting(struct rd *rd, struct nlattr **tb)
+{
+	uint8_t dim_setting;
+
+	if (!tb[RDMA_NLDEV_ATTR_DEV_DIM])
+		return;
+
+	dim_setting = mnl_attr_get_u8(tb[RDMA_NLDEV_ATTR_DEV_DIM]);
+	if (dim_setting > 1)
+		return;
+
+	print_on_off(rd, "adaptive-moderation", dim_setting);
+
+}
+
 static const char *node_type_to_str(uint8_t node_type)
 {
 	static const char * const node_type_str[] = { "unknown", "ca",
@@ -219,8 +235,10 @@ static int dev_parse_cb(const struct nlmsghdr *nlh, void *data)
 	dev_print_fw(rd, tb);
 	dev_print_node_guid(rd, tb);
 	dev_print_sys_image_guid(rd, tb);
-	if (rd->show_details)
+	if (rd->show_details) {
+		dev_print_dim_setting(rd, tb);
 		dev_print_caps(rd, tb);
+	}
 
 	if (!rd->json_output)
 		pr_out("\n");
@@ -308,12 +326,47 @@ done:
 	return ret;
 }
 
+static int dev_set_dim_sendmsg(struct rd *rd, uint8_t dim_setting)
+{
+	uint32_t seq;
+
+	rd_prepare_msg(rd, RDMA_NLDEV_CMD_SET, &seq,
+		       (NLM_F_REQUEST | NLM_F_ACK));
+	mnl_attr_put_u32(rd->nlh, RDMA_NLDEV_ATTR_DEV_INDEX, rd->dev_idx);
+	mnl_attr_put_u8(rd->nlh, RDMA_NLDEV_ATTR_DEV_DIM, dim_setting);
+
+	return rd_sendrecv_msg(rd, seq);
+}
+
+static int dev_set_dim_off(struct rd *rd)
+{
+	return dev_set_dim_sendmsg(rd, 0);
+}
+
+static int dev_set_dim_on(struct rd *rd)
+{
+	return dev_set_dim_sendmsg(rd, 1);
+}
+
+static int dev_set_dim(struct rd *rd)
+{
+	const struct rd_cmd cmds[] = {
+		{ NULL,		dev_help},
+		{ "on",		dev_set_dim_on},
+		{ "off",	dev_set_dim_off},
+		{ 0 }
+	};
+
+	return rd_exec_cmd(rd, cmds, "parameter");
+}
+
 static int dev_one_set(struct rd *rd)
 {
 	const struct rd_cmd cmds[] = {
 		{ NULL,		dev_help},
 		{ "name",	dev_set_name},
 		{ "netns",	dev_set_netns},
+		{ "adaptive-moderation",	dev_set_dim},
 		{ 0 }
 	};
 
diff --git a/rdma/rdma.h b/rdma/rdma.h
index 23157743..dfd1b70b 100644
--- a/rdma/rdma.h
+++ b/rdma/rdma.h
@@ -136,6 +136,7 @@ int rd_attr_check(const struct nlattr *attr, int *typep);
 void print_driver_table(struct rd *rd, struct nlattr *tb);
 void newline(struct rd *rd);
 void newline_indent(struct rd *rd);
+void print_on_off(struct rd *rd, const char *key_str, bool on);
 #define MAX_LINE_LENGTH 80
 
 #endif /* _RDMA_TOOL_H_ */
diff --git a/rdma/res-cq.c b/rdma/res-cq.c
index 5afb97c5..d2591fbe 100644
--- a/rdma/res-cq.c
+++ b/rdma/res-cq.c
@@ -30,6 +30,20 @@ static void print_poll_ctx(struct rd *rd, uint8_t poll_ctx, struct nlattr *attr)
 	pr_out("poll-ctx %s ", poll_ctx_to_str(poll_ctx));
 }
 
+static void print_cq_dim_setting(struct rd *rd, struct nlattr *attr)
+{
+	uint8_t dim_setting;
+
+	if (!attr)
+		return;
+
+	dim_setting = mnl_attr_get_u8(attr);
+	if (dim_setting > 1)
+		return;
+
+	print_on_off(rd, "adaptive-moderation", dim_setting);
+}
+
 static int res_cq_line(struct rd *rd, const char *name, int idx,
 		       struct nlattr **nla_line)
 {
@@ -97,6 +111,7 @@ static int res_cq_line(struct rd *rd, const char *name, int idx,
 	res_print_uint(rd, "users", users,
 		       nla_line[RDMA_NLDEV_ATTR_RES_USECNT]);
 	print_poll_ctx(rd, poll_ctx, nla_line[RDMA_NLDEV_ATTR_RES_POLL_CTX]);
+	print_cq_dim_setting(rd, nla_line[RDMA_NLDEV_ATTR_DEV_DIM]);
 	res_print_uint(rd, "ctxn", ctxn, nla_line[RDMA_NLDEV_ATTR_RES_CTXN]);
 	res_print_uint(rd, "pid", pid, nla_line[RDMA_NLDEV_ATTR_RES_PID]);
 	print_comm(rd, comm, nla_line);
diff --git a/rdma/utils.c b/rdma/utils.c
index 95b669f3..37659011 100644
--- a/rdma/utils.c
+++ b/rdma/utils.c
@@ -449,6 +449,7 @@ static const enum mnl_attr_data_type nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
 	[RDMA_NLDEV_ATTR_STAT_MODE] = MNL_TYPE_U32,
 	[RDMA_NLDEV_ATTR_STAT_RES] = MNL_TYPE_U32,
 	[RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK] = MNL_TYPE_U32,
+	[RDMA_NLDEV_ATTR_DEV_DIM] = MNL_TYPE_U8,
 };
 
 int rd_attr_check(const struct nlattr *attr, int *typep)
@@ -789,6 +790,11 @@ static int print_driver_string(struct rd *rd, const char *key_str,
 	}
 }
 
+void print_on_off(struct rd *rd, const char *key_str, bool on)
+{
+	print_driver_string(rd, key_str, (on) ? "on":"off");
+}
+
 static int print_driver_s32(struct rd *rd, const char *key_str, int32_t val,
 			      enum rdma_nldev_print_type print_type)
 {
-- 
2.20.1

[PATCH iproute2-rc 2/2] rdma: Document adaptive-moderation

[Leon Romanovsky]

From: Yamin Friedman <yaminf@mellanox.com>

Add document of setting the adaptive-moderation for the ib device.

Signed-off-by: Yamin Friedman <yaminf@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
 man/man8/rdma-dev.8 | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/man/man8/rdma-dev.8 b/man/man8/rdma-dev.8
index e77e7cd0..368cdc7c 100644
--- a/man/man8/rdma-dev.8
+++ b/man/man8/rdma-dev.8
@@ -34,11 +34,17 @@ rdma-dev \- RDMA device configuration
 .BR netns
 .BR NSNAME
 
+.ti -8
+.B rdma dev set
+.RI "[ " DEV " ]"
+.BR adaptive-moderation
+.BR [on/off]
+
 .ti -8
 .B rdma dev help
 
 .SH "DESCRIPTION"
-.SS rdma dev set - rename RDMA device or set network namespace
+.SS rdma dev set - rename RDMA device or set network namespace or set RDMA device adaptive-moderation
 
 .SS rdma dev show - display RDMA device attributes
 
@@ -70,6 +76,14 @@ Changes the network namespace of RDMA device to foo where foo is
 previously created using iproute2 ip command.
 .RE
 .PP
+rdma dev set mlx5_3 adaptive-moderation [on/off]
+.RS 4
+Sets the state of adaptive interrupt moderation for the RDMA device.
+.RE
+.RS 4
+This is a global setting for the RDMA device but the value is printed for each CQ individually because the state is constant from CQ allocation.
+.RE
+.PP
 
 .SH SEE ALSO
 .BR ip (8),
-- 
2.20.1