Re: [PATCH v5 2/3] RDMA/rxe: add SENT/RCVD bytes

[Zhu Yanjun <yanjun.zhu@linux.dev>]

在 2026/4/11 0:24, zhenwei pi 写道:
> 
> 
> On 4/11/26 09:45, Zhu Yanjun wrote:
>>
>> 在 2026/4/10 15:37, yanjun.zhu 写道:
>>> On 4/7/26 5:09 PM, zhenwei pi wrote:
>>>> There is a lack of sent/received counter in bytes.
>>>>
>>>> Signed-off-by: zhenwei pi <zhenwei.pi@linux.dev>
>>>> ---
>>>>   drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++
>>>>   drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++
>>>>   drivers/infiniband/sw/rxe/rxe_net.c         | 2 ++
>>>>   drivers/infiniband/sw/rxe/rxe_recv.c        | 2 ++
>>>>   drivers/infiniband/sw/rxe/rxe_verbs.h       | 6 ++++++
>>>>   5 files changed, 14 insertions(+)
>>>>
>>>> diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.c b/drivers/ 
>>>> infiniband/sw/rxe/rxe_hw_counters.c
>>>> index 437917a7d8f2..17edaa9a9b9b 100644
>>>> --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.c
>>>> +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
>>>> @@ -22,6 +22,8 @@ static const struct rdma_stat_desc 
>>>> rxe_counter_descs[] = {
>>>>       [RXE_CNT_LINK_DOWNED].name         =  "link_downed",
>>>>       [RXE_CNT_RDMA_SEND].name           =  "rdma_sends",
>>>>       [RXE_CNT_RDMA_RECV].name           =  "rdma_recvs",
>>>> +    [RXE_CNT_SENT_BYTES].name          =  "sent_bytes",
>>>> +    [RXE_CNT_RCVD_BYTES].name          =  "rcvd_bytes",
>>>>   };
>>>>     int rxe_ib_get_hw_stats(struct ib_device *ibdev,
>>>> diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.h b/drivers/ 
>>>> infiniband/sw/rxe/rxe_hw_counters.h
>>>> index 051f9e1c3852..01b355103cbc 100644
>>>> --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.h
>>>> +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.h
>>>> @@ -26,6 +26,8 @@ enum rxe_counters {
>>>>       RXE_CNT_LINK_DOWNED,
>>>>       RXE_CNT_RDMA_SEND,
>>>>       RXE_CNT_RDMA_RECV,
>>>> +    RXE_CNT_SENT_BYTES,
>>>> +    RXE_CNT_RCVD_BYTES,
>>>>       RXE_NUM_OF_COUNTERS
>>>>   };
>>>>   diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/ 
>>>> infiniband/sw/rxe/rxe_net.c
>>>> index 6621d01ac32d..86660031ffa2 100644
>>>> --- a/drivers/infiniband/sw/rxe/rxe_net.c
>>>> +++ b/drivers/infiniband/sw/rxe/rxe_net.c
>>>> @@ -503,6 +503,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct 
>>>> rxe_pkt_info *pkt,
>>>>       int err;
>>>>       int is_request = pkt->mask & RXE_REQ_MASK;
>>>>       struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
>>>> +    unsigned int skblen = skb->len;
>>>>       unsigned long flags;
>>>>         spin_lock_irqsave(&qp->state_lock, flags);
>>>> @@ -526,6 +527,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct 
>>>> rxe_pkt_info *pkt,
>>>>       }
>>>>         rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
>>>> +    rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skblen);
>>>>       goto done;
>>>>     drop:
>>>> diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/ 
>>>> infiniband/sw/rxe/rxe_recv.c
>>>> index 5861e4244049..e7bab89e7d8d 100644
>>>> --- a/drivers/infiniband/sw/rxe/rxe_recv.c
>>>> +++ b/drivers/infiniband/sw/rxe/rxe_recv.c
>>>> @@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb)
>>>>       int err;
>>>>       struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
>>>>       struct rxe_dev *rxe = pkt->rxe;
>>>> +    unsigned int skblen = skb->len - skb_network_offset(skb);
>>>>         if (unlikely(skb->len < RXE_BTH_BYTES))
>>>>           goto drop;
>>>> @@ -341,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb)
>>>>       if (unlikely(err))
>>>>           goto drop;
>>>>   +    rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);
>>>>       rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS);
>>>>         if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))
>>>
>>> int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
>>>                     struct sk_buff *skb)
>>> {
>>>         int err;
>>>         int is_request = pkt->mask & RXE_REQ_MASK;
>>>         struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
>>>         unsigned long flags;
>>>
>>> skb->len is printed here, that is len1
>>> ...
>>>         if (pkt->mask & RXE_LOOPBACK_MASK)
>>>                 err = rxe_loopback(skb, pkt);
>>>         else
>>>                 err = rxe_send(skb, pkt);
>>> ...
>>> }
>>>
>>> In the following function
>>>
>>> static int rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt)
>>> {
>>> ...
>>>         if (skb->protocol == htons(ETH_P_IP))
>>>                 skb_pull(skb, sizeof(struct iphdr));
>>>         else
>>>                 skb_pull(skb, sizeof(struct ipv6hdr));
>>>
>>> ...
>>>         /* remove udp header */
>>>         skb_pull(skb, sizeof(struct udphdr));
>>>
>>> print skb->len here, that is len2
>>>
>>>         rxe_rcv(skb);
>>>
>>> ...
>>> }
>>>
>>> Does len1 equal to len2?
>>
>> I have made tests. The difference between len1 and len2 is 28.
>>
>> It should be the total of ipv4 header + udp header because I use ipv4 
>> address to make tests.
>>
>> I am not sure if the bytes of recv data should equal to the bytes of 
>> xmit data.
>>
>>
>> Zhu Yanjun
>>
> 
> Yes, I got the same result of 28 bytes difference during v4 version. So 
> I try to use such logic to recalculate the length:
> 
> +    unsigned int skblen = skb->len + sizeof(struct udphdr);
> ...
> +    if (skb->protocol == htons(ETH_P_IP))
> +        skblen += sizeof(struct iphdr);
> +    else if (skb->protocol == htons(ETH_P_IPV6))
> +        skblen += sizeof(struct ipv6hdr);
> +    rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);
> 
> then use 'unsigned int skblen = skb->len - skb_network_offset(skb);' in v5.

Thanks. Not necessary to use this 'unsigned int skblen = skb->len - 
skb_network_offset(skb);'.

The above udphdr and iphdr/ip6hdr should be OK.

Thanks a lot.
Zhu Yanjun

> 
>>>
>>> If not, the transmitted length appears to differ from the received 
>>> length when using loopback.
>>>
>>> I am not sure whether this is expected behavior.
>>>
>>> The same observation also applies to the non-loopback case.
>>>
>>> Zhu Yanjun
>>>
>>>> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/ 
>>>> infiniband/sw/rxe/rxe_verbs.h
>>>> index e800545d1046..0f5ffd94643f 100644
>>>> --- a/drivers/infiniband/sw/rxe/rxe_verbs.h
>>>> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
>>>> @@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct 
>>>> rxe_dev *rxe, enum rxe_counters index)
>>>>       atomic64_inc(&rxe->stats_counters[index]);
>>>>   }
>>>>   +static inline void rxe_counter_add(struct rxe_dev *rxe, enum 
>>>> rxe_counters index,
>>>> +                   s64 val)
>>>> +{
>>>> +    atomic64_add(val, &rxe->stats_counters[index]);
>>>> +}
>>>> +
>>>>   static inline struct rxe_dev *to_rdev(struct ib_device *dev)
>>>>   {
>>>>       return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
>>
>