* [PATCH v3 1/3] RDMA/rxe: use RXE_PORT instead of magic number 1
2026-03-29 5:41 [PATCH v3 0/3] Support PERF MGMT for RXE zhenwei pi
@ 2026-03-29 5:41 ` zhenwei pi
2026-03-29 7:01 ` Zhu Yanjun
2026-03-29 5:41 ` [PATCH v3 2/3] RDMA/rxe: add SENT/RCVD bytes zhenwei pi
2026-03-29 5:41 ` [PATCH v3 3/3] RDMA/rxe: support perf mgmt GET method zhenwei pi
2 siblings, 1 reply; 7+ messages in thread
From: zhenwei pi @ 2026-03-29 5:41 UTC (permalink / raw)
To: linux-kernel, linux-rdma; +Cc: zyjzyj2000, jgg, leon, zhenwei pi
Align with the existing code:
static ... rxe_ib_device_get_netdev(struct ib_device *dev)
{
return ib_device_get_netdev(dev, RXE_PORT);
}
Use *RXE_PORT* instead of magic number 1 for all.
Signed-off-by: zhenwei pi <zhenwei.pi@linux.dev>
---
drivers/infiniband/sw/rxe/rxe_net.c | 6 +++---
drivers/infiniband/sw/rxe/rxe_verbs.c | 8 ++++----
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index 0bd0902b11f7..20338cb8e3c2 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -234,7 +234,7 @@ static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
udph = udp_hdr(skb);
pkt->rxe = rxe;
- pkt->port_num = 1;
+ pkt->port_num = RXE_PORT;
pkt->hdr = (u8 *)(udph + 1);
pkt->mask = RXE_GRH_MASK;
pkt->paylen = be16_to_cpu(udph->len) - sizeof(*udph);
@@ -535,7 +535,7 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
struct sk_buff *skb = NULL;
struct net_device *ndev;
const struct ib_gid_attr *attr;
- const int port_num = 1;
+ const int port_num = RXE_PORT;
attr = rdma_get_gid_attr(&rxe->ib_dev, port_num, av->grh.sgid_index);
if (IS_ERR(attr))
@@ -630,7 +630,7 @@ static void rxe_port_event(struct rxe_dev *rxe,
struct ib_event ev;
ev.device = &rxe->ib_dev;
- ev.element.port_num = 1;
+ ev.element.port_num = RXE_PORT;
ev.event = event;
ib_dispatch_event(&ev);
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index fe41362c5144..bcd486e8668b 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -44,7 +44,7 @@ static int rxe_query_port(struct ib_device *ibdev,
struct net_device *ndev;
int err, ret;
- if (port_num != 1) {
+ if (port_num != RXE_PORT) {
err = -EINVAL;
rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
goto err_out;
@@ -147,7 +147,7 @@ static int rxe_modify_port(struct ib_device *ibdev, u32 port_num,
struct rxe_port *port;
int err;
- if (port_num != 1) {
+ if (port_num != RXE_PORT) {
err = -EINVAL;
rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
goto err_out;
@@ -180,7 +180,7 @@ static enum rdma_link_layer rxe_get_link_layer(struct ib_device *ibdev,
struct rxe_dev *rxe = to_rdev(ibdev);
int err;
- if (port_num != 1) {
+ if (port_num != RXE_PORT) {
err = -EINVAL;
rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
goto err_out;
@@ -200,7 +200,7 @@ static int rxe_port_immutable(struct ib_device *ibdev, u32 port_num,
struct ib_port_attr attr = {};
int err;
- if (port_num != 1) {
+ if (port_num != RXE_PORT) {
err = -EINVAL;
rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
goto err_out;
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: [PATCH v3 1/3] RDMA/rxe: use RXE_PORT instead of magic number 1
2026-03-29 5:41 ` [PATCH v3 1/3] RDMA/rxe: use RXE_PORT instead of magic number 1 zhenwei pi
@ 2026-03-29 7:01 ` Zhu Yanjun
0 siblings, 0 replies; 7+ messages in thread
From: Zhu Yanjun @ 2026-03-29 7:01 UTC (permalink / raw)
To: zhenwei pi, linux-kernel, linux-rdma, yanjun.zhu@linux.dev
Cc: zyjzyj2000, jgg, leon
在 2026/3/28 22:41, zhenwei pi 写道:
> Align with the existing code:
> static ... rxe_ib_device_get_netdev(struct ib_device *dev)
> {
> return ib_device_get_netdev(dev, RXE_PORT);
> }
>
> Use *RXE_PORT* instead of magic number 1 for all.
>
Thanks a lot.
Reviewed-by: Zhu Yanjun <yanjun.zhu@linux.dev>
Zhu Yanjun
> Signed-off-by: zhenwei pi <zhenwei.pi@linux.dev>
> ---
> drivers/infiniband/sw/rxe/rxe_net.c | 6 +++---
> drivers/infiniband/sw/rxe/rxe_verbs.c | 8 ++++----
> 2 files changed, 7 insertions(+), 7 deletions(-)
>
> diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
> index 0bd0902b11f7..20338cb8e3c2 100644
> --- a/drivers/infiniband/sw/rxe/rxe_net.c
> +++ b/drivers/infiniband/sw/rxe/rxe_net.c
> @@ -234,7 +234,7 @@ static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
>
> udph = udp_hdr(skb);
> pkt->rxe = rxe;
> - pkt->port_num = 1;
> + pkt->port_num = RXE_PORT;
> pkt->hdr = (u8 *)(udph + 1);
> pkt->mask = RXE_GRH_MASK;
> pkt->paylen = be16_to_cpu(udph->len) - sizeof(*udph);
> @@ -535,7 +535,7 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
> struct sk_buff *skb = NULL;
> struct net_device *ndev;
> const struct ib_gid_attr *attr;
> - const int port_num = 1;
> + const int port_num = RXE_PORT;
>
> attr = rdma_get_gid_attr(&rxe->ib_dev, port_num, av->grh.sgid_index);
> if (IS_ERR(attr))
> @@ -630,7 +630,7 @@ static void rxe_port_event(struct rxe_dev *rxe,
> struct ib_event ev;
>
> ev.device = &rxe->ib_dev;
> - ev.element.port_num = 1;
> + ev.element.port_num = RXE_PORT;
> ev.event = event;
>
> ib_dispatch_event(&ev);
> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
> index fe41362c5144..bcd486e8668b 100644
> --- a/drivers/infiniband/sw/rxe/rxe_verbs.c
> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
> @@ -44,7 +44,7 @@ static int rxe_query_port(struct ib_device *ibdev,
> struct net_device *ndev;
> int err, ret;
>
> - if (port_num != 1) {
> + if (port_num != RXE_PORT) {
> err = -EINVAL;
> rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
> goto err_out;
> @@ -147,7 +147,7 @@ static int rxe_modify_port(struct ib_device *ibdev, u32 port_num,
> struct rxe_port *port;
> int err;
>
> - if (port_num != 1) {
> + if (port_num != RXE_PORT) {
> err = -EINVAL;
> rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
> goto err_out;
> @@ -180,7 +180,7 @@ static enum rdma_link_layer rxe_get_link_layer(struct ib_device *ibdev,
> struct rxe_dev *rxe = to_rdev(ibdev);
> int err;
>
> - if (port_num != 1) {
> + if (port_num != RXE_PORT) {
> err = -EINVAL;
> rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
> goto err_out;
> @@ -200,7 +200,7 @@ static int rxe_port_immutable(struct ib_device *ibdev, u32 port_num,
> struct ib_port_attr attr = {};
> int err;
>
> - if (port_num != 1) {
> + if (port_num != RXE_PORT) {
> err = -EINVAL;
> rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
> goto err_out;
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH v3 2/3] RDMA/rxe: add SENT/RCVD bytes
2026-03-29 5:41 [PATCH v3 0/3] Support PERF MGMT for RXE zhenwei pi
2026-03-29 5:41 ` [PATCH v3 1/3] RDMA/rxe: use RXE_PORT instead of magic number 1 zhenwei pi
@ 2026-03-29 5:41 ` zhenwei pi
2026-03-29 7:01 ` Zhu Yanjun
2026-03-29 5:41 ` [PATCH v3 3/3] RDMA/rxe: support perf mgmt GET method zhenwei pi
2 siblings, 1 reply; 7+ messages in thread
From: zhenwei pi @ 2026-03-29 5:41 UTC (permalink / raw)
To: linux-kernel, linux-rdma; +Cc: zyjzyj2000, jgg, leon, zhenwei pi
There is a lack of sent/received counter in bytes.
Signed-off-by: zhenwei pi <zhenwei.pi@linux.dev>
---
drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++
drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++
drivers/infiniband/sw/rxe/rxe_net.c | 1 +
drivers/infiniband/sw/rxe/rxe_recv.c | 1 +
drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++
5 files changed, 12 insertions(+)
diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.c b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
index 437917a7d8f2..17edaa9a9b9b 100644
--- a/drivers/infiniband/sw/rxe/rxe_hw_counters.c
+++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = {
[RXE_CNT_LINK_DOWNED].name = "link_downed",
[RXE_CNT_RDMA_SEND].name = "rdma_sends",
[RXE_CNT_RDMA_RECV].name = "rdma_recvs",
+ [RXE_CNT_SENT_BYTES].name = "sent_bytes",
+ [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes",
};
int rxe_ib_get_hw_stats(struct ib_device *ibdev,
diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.h b/drivers/infiniband/sw/rxe/rxe_hw_counters.h
index 051f9e1c3852..01b355103cbc 100644
--- a/drivers/infiniband/sw/rxe/rxe_hw_counters.h
+++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.h
@@ -26,6 +26,8 @@ enum rxe_counters {
RXE_CNT_LINK_DOWNED,
RXE_CNT_RDMA_SEND,
RXE_CNT_RDMA_RECV,
+ RXE_CNT_SENT_BYTES,
+ RXE_CNT_RCVD_BYTES,
RXE_NUM_OF_COUNTERS
};
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index 20338cb8e3c2..ec0ae7479fe7 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -519,6 +519,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
}
rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
+ rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skb->len);
goto done;
drop:
diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c
index 5861e4244049..b5522017852d 100644
--- a/drivers/infiniband/sw/rxe/rxe_recv.c
+++ b/drivers/infiniband/sw/rxe/rxe_recv.c
@@ -342,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb)
goto drop;
rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS);
+ rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skb->len);
if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))
rxe_rcv_mcast_pkt(rxe, skb);
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index fb149f37e91d..2bcfb919a40b 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -460,6 +460,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
atomic64_inc(&rxe->stats_counters[index]);
}
+static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,
+ s64 val)
+{
+ atomic64_add(val, &rxe->stats_counters[index]);
+}
+
static inline struct rxe_dev *to_rdev(struct ib_device *dev)
{
return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: [PATCH v3 2/3] RDMA/rxe: add SENT/RCVD bytes
2026-03-29 5:41 ` [PATCH v3 2/3] RDMA/rxe: add SENT/RCVD bytes zhenwei pi
@ 2026-03-29 7:01 ` Zhu Yanjun
0 siblings, 0 replies; 7+ messages in thread
From: Zhu Yanjun @ 2026-03-29 7:01 UTC (permalink / raw)
To: zhenwei pi, linux-kernel, linux-rdma, yanjun.zhu@linux.dev
Cc: zyjzyj2000, jgg, leon
在 2026/3/28 22:41, zhenwei pi 写道:
> There is a lack of sent/received counter in bytes.
>
Thanks a lot.
Reviewed-by: Zhu Yanjun <yanjun.zhu@linux.dev>
Zhu Yanjun
> Signed-off-by: zhenwei pi <zhenwei.pi@linux.dev>
> ---
> drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++
> drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++
> drivers/infiniband/sw/rxe/rxe_net.c | 1 +
> drivers/infiniband/sw/rxe/rxe_recv.c | 1 +
> drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++
> 5 files changed, 12 insertions(+)
>
> diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.c b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
> index 437917a7d8f2..17edaa9a9b9b 100644
> --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.c
> +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
> @@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = {
> [RXE_CNT_LINK_DOWNED].name = "link_downed",
> [RXE_CNT_RDMA_SEND].name = "rdma_sends",
> [RXE_CNT_RDMA_RECV].name = "rdma_recvs",
> + [RXE_CNT_SENT_BYTES].name = "sent_bytes",
> + [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes",
> };
>
> int rxe_ib_get_hw_stats(struct ib_device *ibdev,
> diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.h b/drivers/infiniband/sw/rxe/rxe_hw_counters.h
> index 051f9e1c3852..01b355103cbc 100644
> --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.h
> +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.h
> @@ -26,6 +26,8 @@ enum rxe_counters {
> RXE_CNT_LINK_DOWNED,
> RXE_CNT_RDMA_SEND,
> RXE_CNT_RDMA_RECV,
> + RXE_CNT_SENT_BYTES,
> + RXE_CNT_RCVD_BYTES,
> RXE_NUM_OF_COUNTERS
> };
>
> diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
> index 20338cb8e3c2..ec0ae7479fe7 100644
> --- a/drivers/infiniband/sw/rxe/rxe_net.c
> +++ b/drivers/infiniband/sw/rxe/rxe_net.c
> @@ -519,6 +519,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
> }
>
> rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
> + rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skb->len);
> goto done;
>
> drop:
> diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c
> index 5861e4244049..b5522017852d 100644
> --- a/drivers/infiniband/sw/rxe/rxe_recv.c
> +++ b/drivers/infiniband/sw/rxe/rxe_recv.c
> @@ -342,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb)
> goto drop;
>
> rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS);
> + rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skb->len);
>
> if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))
> rxe_rcv_mcast_pkt(rxe, skb);
> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
> index fb149f37e91d..2bcfb919a40b 100644
> --- a/drivers/infiniband/sw/rxe/rxe_verbs.h
> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
> @@ -460,6 +460,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
> atomic64_inc(&rxe->stats_counters[index]);
> }
>
> +static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,
> + s64 val)
> +{
> + atomic64_add(val, &rxe->stats_counters[index]);
> +}
> +
> static inline struct rxe_dev *to_rdev(struct ib_device *dev)
> {
> return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH v3 3/3] RDMA/rxe: support perf mgmt GET method
2026-03-29 5:41 [PATCH v3 0/3] Support PERF MGMT for RXE zhenwei pi
2026-03-29 5:41 ` [PATCH v3 1/3] RDMA/rxe: use RXE_PORT instead of magic number 1 zhenwei pi
2026-03-29 5:41 ` [PATCH v3 2/3] RDMA/rxe: add SENT/RCVD bytes zhenwei pi
@ 2026-03-29 5:41 ` zhenwei pi
2026-03-29 7:03 ` Zhu Yanjun
2 siblings, 1 reply; 7+ messages in thread
From: zhenwei pi @ 2026-03-29 5:41 UTC (permalink / raw)
To: linux-kernel, linux-rdma; +Cc: zyjzyj2000, jgg, leon, zhenwei pi
In RXE, hardware counters are already supported, but not in a
standardized manner. For instance, user-space monitoring tools like
atop only read from the *counters* directory. Therefore, it is
necessary to add perf management support to RXE.
Also use rxe_counter_get instead of raw atomic64_read in hw-counters.
Signed-off-by: zhenwei pi <zhenwei.pi@linux.dev>
---
drivers/infiniband/sw/rxe/Makefile | 1 +
drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 +-
drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++
drivers/infiniband/sw/rxe/rxe_mad.c | 93 +++++++++++++++++++++
drivers/infiniband/sw/rxe/rxe_verbs.c | 1 +
drivers/infiniband/sw/rxe/rxe_verbs.h | 5 ++
6 files changed, 107 insertions(+), 1 deletion(-)
create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c
diff --git a/drivers/infiniband/sw/rxe/Makefile b/drivers/infiniband/sw/rxe/Makefile
index 93134f1d1d0c..3c47e5b982c2 100644
--- a/drivers/infiniband/sw/rxe/Makefile
+++ b/drivers/infiniband/sw/rxe/Makefile
@@ -22,6 +22,7 @@ rdma_rxe-y := \
rxe_mcast.o \
rxe_task.o \
rxe_net.o \
+ rxe_mad.o \
rxe_hw_counters.o
rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.c b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
index 17edaa9a9b9b..a612e96f7a88 100644
--- a/drivers/infiniband/sw/rxe/rxe_hw_counters.c
+++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
@@ -37,7 +37,7 @@ int rxe_ib_get_hw_stats(struct ib_device *ibdev,
return -EINVAL;
for (cnt = 0; cnt < ARRAY_SIZE(rxe_counter_descs); cnt++)
- stats->value[cnt] = atomic64_read(&dev->stats_counters[cnt]);
+ stats->value[cnt] = rxe_counter_get(dev, cnt);
return ARRAY_SIZE(rxe_counter_descs);
}
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index 7992290886e1..a8ce85147c1f 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -245,4 +245,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd,
#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
+/* rxe-mad.c */
+int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,
+ const struct ib_wc *in_wc, const struct ib_grh *in_grh,
+ const struct ib_mad *in, struct ib_mad *out,
+ size_t *out_mad_size, u16 *out_mad_pkey_index);
+
#endif /* RXE_LOC_H */
diff --git a/drivers/infiniband/sw/rxe/rxe_mad.c b/drivers/infiniband/sw/rxe/rxe_mad.c
new file mode 100644
index 000000000000..f41b8d5cdfe1
--- /dev/null
+++ b/drivers/infiniband/sw/rxe/rxe_mad.c
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright (c) 2026 zhenwei pi <zhenwei.pi@linux.dev>
+ */
+
+#include <rdma/ib_pma.h>
+#include "rxe.h"
+#include "rxe_hw_counters.h"
+
+static int rxe_get_pma_info(struct ib_mad *out)
+{
+ struct ib_class_port_info cpi = {};
+
+ cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH;
+ memcpy((out->data + 40), &cpi, sizeof(cpi));
+
+ return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+}
+
+static int rxe_get_pma_counters(struct rxe_dev *rxe, struct ib_mad *out)
+{
+ struct ib_pma_portcounters *pma_cnt = (struct ib_pma_portcounters *)(out->data + 40);
+ s64 val;
+
+ /* IBA release 1.8, 16.1.3.5: During operation, instead of overflowing, they shall stop
+ * at all ones.
+ */
+ val = rxe_counter_get(rxe, RXE_CNT_LINK_DOWNED);
+ if (val > U8_MAX)
+ pma_cnt->link_downed_counter = U8_MAX;
+ else
+ pma_cnt->link_downed_counter = (u8)val;
+
+ return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+}
+
+static int rxe_get_pma_counters_ext(struct rxe_dev *rxe, struct ib_mad *out)
+{
+ struct ib_pma_portcounters_ext *pma_cnt_ext =
+ (struct ib_pma_portcounters_ext *)(out->data + 40);
+
+ pma_cnt_ext->port_xmit_data = cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_SENT_BYTES) >> 2);
+ pma_cnt_ext->port_rcv_data = cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_RCVD_BYTES) >> 2);
+ pma_cnt_ext->port_xmit_packets = cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_SENT_PKTS));
+ pma_cnt_ext->port_rcv_packets = cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_RCVD_PKTS));
+
+ return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+}
+
+static int rxe_get_perf_mgmt(struct rxe_dev *rxe, const struct ib_mad *in, struct ib_mad *out)
+{
+ switch (in->mad_hdr.attr_id) {
+ case IB_PMA_CLASS_PORT_INFO:
+ return rxe_get_pma_info(out);
+
+ case IB_PMA_PORT_COUNTERS:
+ return rxe_get_pma_counters(rxe, out);
+
+ case IB_PMA_PORT_COUNTERS_EXT:
+ return rxe_get_pma_counters_ext(rxe, out);
+
+ default:
+ out->mad_hdr.status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB);
+ return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+ }
+}
+
+int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,
+ const struct ib_wc *in_wc, const struct ib_grh *in_grh,
+ const struct ib_mad *in, struct ib_mad *out,
+ size_t *out_mad_size, u16 *out_mad_pkey_index)
+{
+ struct rxe_dev *rxe = to_rdev(ibdev);
+ u8 mgmt_class = in->mad_hdr.mgmt_class;
+ u8 method = in->mad_hdr.method;
+
+ if (port_num != RXE_PORT)
+ return IB_MAD_RESULT_FAILURE;
+
+ memset(out, 0, sizeof(*out));
+ switch (mgmt_class) {
+ case IB_MGMT_CLASS_PERF_MGMT:
+ if (method == IB_MGMT_METHOD_GET)
+ return rxe_get_perf_mgmt(rxe, in, out);
+ break;
+
+ default:
+ break;
+ }
+
+ out->mad_hdr.status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD);
+ return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+}
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index bcd486e8668b..7df0cb5a09a3 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -1509,6 +1509,7 @@ static const struct ib_device_ops rxe_dev_ops = {
.post_recv = rxe_post_recv,
.post_send = rxe_post_send,
.post_srq_recv = rxe_post_srq_recv,
+ .process_mad = rxe_process_mad,
.query_ah = rxe_query_ah,
.query_device = rxe_query_device,
.query_pkey = rxe_query_pkey,
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 2bcfb919a40b..1c4fa8eaa733 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -466,6 +466,11 @@ static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,
atomic64_add(val, &rxe->stats_counters[index]);
}
+static inline s64 rxe_counter_get(struct rxe_dev *rxe, enum rxe_counters index)
+{
+ return atomic64_read(&rxe->stats_counters[index]);
+}
+
static inline struct rxe_dev *to_rdev(struct ib_device *dev)
{
return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: [PATCH v3 3/3] RDMA/rxe: support perf mgmt GET method
2026-03-29 5:41 ` [PATCH v3 3/3] RDMA/rxe: support perf mgmt GET method zhenwei pi
@ 2026-03-29 7:03 ` Zhu Yanjun
0 siblings, 0 replies; 7+ messages in thread
From: Zhu Yanjun @ 2026-03-29 7:03 UTC (permalink / raw)
To: zhenwei pi, linux-kernel, linux-rdma, yanjun.zhu@linux.dev
Cc: zyjzyj2000, jgg, leon
在 2026/3/28 22:41, zhenwei pi 写道:
> In RXE, hardware counters are already supported, but not in a
> standardized manner. For instance, user-space monitoring tools like
> atop only read from the *counters* directory. Therefore, it is
> necessary to add perf management support to RXE.
>
> Also use rxe_counter_get instead of raw atomic64_read in hw-counters.
>
Thanks a lot.
Reviewed-by: Zhu Yanjun <yanjun.zhu@linux.dev>
Zhu Yanjun
> Signed-off-by: zhenwei pi <zhenwei.pi@linux.dev>
> ---
> drivers/infiniband/sw/rxe/Makefile | 1 +
> drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 +-
> drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++
> drivers/infiniband/sw/rxe/rxe_mad.c | 93 +++++++++++++++++++++
> drivers/infiniband/sw/rxe/rxe_verbs.c | 1 +
> drivers/infiniband/sw/rxe/rxe_verbs.h | 5 ++
> 6 files changed, 107 insertions(+), 1 deletion(-)
> create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c
>
> diff --git a/drivers/infiniband/sw/rxe/Makefile b/drivers/infiniband/sw/rxe/Makefile
> index 93134f1d1d0c..3c47e5b982c2 100644
> --- a/drivers/infiniband/sw/rxe/Makefile
> +++ b/drivers/infiniband/sw/rxe/Makefile
> @@ -22,6 +22,7 @@ rdma_rxe-y := \
> rxe_mcast.o \
> rxe_task.o \
> rxe_net.o \
> + rxe_mad.o \
> rxe_hw_counters.o
>
> rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
> diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.c b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
> index 17edaa9a9b9b..a612e96f7a88 100644
> --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.c
> +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
> @@ -37,7 +37,7 @@ int rxe_ib_get_hw_stats(struct ib_device *ibdev,
> return -EINVAL;
>
> for (cnt = 0; cnt < ARRAY_SIZE(rxe_counter_descs); cnt++)
> - stats->value[cnt] = atomic64_read(&dev->stats_counters[cnt]);
> + stats->value[cnt] = rxe_counter_get(dev, cnt);
>
> return ARRAY_SIZE(rxe_counter_descs);
> }
> diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
> index 7992290886e1..a8ce85147c1f 100644
> --- a/drivers/infiniband/sw/rxe/rxe_loc.h
> +++ b/drivers/infiniband/sw/rxe/rxe_loc.h
> @@ -245,4 +245,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd,
>
> #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
>
> +/* rxe-mad.c */
> +int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,
> + const struct ib_wc *in_wc, const struct ib_grh *in_grh,
> + const struct ib_mad *in, struct ib_mad *out,
> + size_t *out_mad_size, u16 *out_mad_pkey_index);
> +
> #endif /* RXE_LOC_H */
> diff --git a/drivers/infiniband/sw/rxe/rxe_mad.c b/drivers/infiniband/sw/rxe/rxe_mad.c
> new file mode 100644
> index 000000000000..f41b8d5cdfe1
> --- /dev/null
> +++ b/drivers/infiniband/sw/rxe/rxe_mad.c
> @@ -0,0 +1,93 @@
> +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
> +/*
> + * Copyright (c) 2026 zhenwei pi <zhenwei.pi@linux.dev>
> + */
> +
> +#include <rdma/ib_pma.h>
> +#include "rxe.h"
> +#include "rxe_hw_counters.h"
> +
> +static int rxe_get_pma_info(struct ib_mad *out)
> +{
> + struct ib_class_port_info cpi = {};
> +
> + cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH;
> + memcpy((out->data + 40), &cpi, sizeof(cpi));
> +
> + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
> +}
> +
> +static int rxe_get_pma_counters(struct rxe_dev *rxe, struct ib_mad *out)
> +{
> + struct ib_pma_portcounters *pma_cnt = (struct ib_pma_portcounters *)(out->data + 40);
> + s64 val;
> +
> + /* IBA release 1.8, 16.1.3.5: During operation, instead of overflowing, they shall stop
> + * at all ones.
> + */
> + val = rxe_counter_get(rxe, RXE_CNT_LINK_DOWNED);
> + if (val > U8_MAX)
> + pma_cnt->link_downed_counter = U8_MAX;
> + else
> + pma_cnt->link_downed_counter = (u8)val;
> +
> + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
> +}
> +
> +static int rxe_get_pma_counters_ext(struct rxe_dev *rxe, struct ib_mad *out)
> +{
> + struct ib_pma_portcounters_ext *pma_cnt_ext =
> + (struct ib_pma_portcounters_ext *)(out->data + 40);
> +
> + pma_cnt_ext->port_xmit_data = cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_SENT_BYTES) >> 2);
> + pma_cnt_ext->port_rcv_data = cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_RCVD_BYTES) >> 2);
> + pma_cnt_ext->port_xmit_packets = cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_SENT_PKTS));
> + pma_cnt_ext->port_rcv_packets = cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_RCVD_PKTS));
> +
> + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
> +}
> +
> +static int rxe_get_perf_mgmt(struct rxe_dev *rxe, const struct ib_mad *in, struct ib_mad *out)
> +{
> + switch (in->mad_hdr.attr_id) {
> + case IB_PMA_CLASS_PORT_INFO:
> + return rxe_get_pma_info(out);
> +
> + case IB_PMA_PORT_COUNTERS:
> + return rxe_get_pma_counters(rxe, out);
> +
> + case IB_PMA_PORT_COUNTERS_EXT:
> + return rxe_get_pma_counters_ext(rxe, out);
> +
> + default:
> + out->mad_hdr.status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB);
> + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
> + }
> +}
> +
> +int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,
> + const struct ib_wc *in_wc, const struct ib_grh *in_grh,
> + const struct ib_mad *in, struct ib_mad *out,
> + size_t *out_mad_size, u16 *out_mad_pkey_index)
> +{
> + struct rxe_dev *rxe = to_rdev(ibdev);
> + u8 mgmt_class = in->mad_hdr.mgmt_class;
> + u8 method = in->mad_hdr.method;
> +
> + if (port_num != RXE_PORT)
> + return IB_MAD_RESULT_FAILURE;
> +
> + memset(out, 0, sizeof(*out));
> + switch (mgmt_class) {
> + case IB_MGMT_CLASS_PERF_MGMT:
> + if (method == IB_MGMT_METHOD_GET)
> + return rxe_get_perf_mgmt(rxe, in, out);
> + break;
> +
> + default:
> + break;
> + }
> +
> + out->mad_hdr.status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD);
> + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
> +}
> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
> index bcd486e8668b..7df0cb5a09a3 100644
> --- a/drivers/infiniband/sw/rxe/rxe_verbs.c
> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
> @@ -1509,6 +1509,7 @@ static const struct ib_device_ops rxe_dev_ops = {
> .post_recv = rxe_post_recv,
> .post_send = rxe_post_send,
> .post_srq_recv = rxe_post_srq_recv,
> + .process_mad = rxe_process_mad,
> .query_ah = rxe_query_ah,
> .query_device = rxe_query_device,
> .query_pkey = rxe_query_pkey,
> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
> index 2bcfb919a40b..1c4fa8eaa733 100644
> --- a/drivers/infiniband/sw/rxe/rxe_verbs.h
> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
> @@ -466,6 +466,11 @@ static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,
> atomic64_add(val, &rxe->stats_counters[index]);
> }
>
> +static inline s64 rxe_counter_get(struct rxe_dev *rxe, enum rxe_counters index)
> +{
> + return atomic64_read(&rxe->stats_counters[index]);
> +}
> +
> static inline struct rxe_dev *to_rdev(struct ib_device *dev)
> {
> return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
^ permalink raw reply [flat|nested] 7+ messages in thread