Re: [PATCH v5 2/3] RDMA/rxe: add SENT/RCVD bytes

[zhenwei pi <zhenwei.pi@linux.dev>]

On 4/11/26 09:45, Zhu Yanjun wrote:
> 
> 在 2026/4/10 15:37, yanjun.zhu 写道:
>> On 4/7/26 5:09 PM, zhenwei pi wrote:
>>> There is a lack of sent/received counter in bytes.
>>>
>>> Signed-off-by: zhenwei pi <zhenwei.pi@linux.dev>
>>> ---
>>>   drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++
>>>   drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++
>>>   drivers/infiniband/sw/rxe/rxe_net.c         | 2 ++
>>>   drivers/infiniband/sw/rxe/rxe_recv.c        | 2 ++
>>>   drivers/infiniband/sw/rxe/rxe_verbs.h       | 6 ++++++
>>>   5 files changed, 14 insertions(+)
>>>
>>> diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.c b/drivers/ 
>>> infiniband/sw/rxe/rxe_hw_counters.c
>>> index 437917a7d8f2..17edaa9a9b9b 100644
>>> --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.c
>>> +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
>>> @@ -22,6 +22,8 @@ static const struct rdma_stat_desc 
>>> rxe_counter_descs[] = {
>>>       [RXE_CNT_LINK_DOWNED].name         =  "link_downed",
>>>       [RXE_CNT_RDMA_SEND].name           =  "rdma_sends",
>>>       [RXE_CNT_RDMA_RECV].name           =  "rdma_recvs",
>>> +    [RXE_CNT_SENT_BYTES].name          =  "sent_bytes",
>>> +    [RXE_CNT_RCVD_BYTES].name          =  "rcvd_bytes",
>>>   };
>>>     int rxe_ib_get_hw_stats(struct ib_device *ibdev,
>>> diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.h b/drivers/ 
>>> infiniband/sw/rxe/rxe_hw_counters.h
>>> index 051f9e1c3852..01b355103cbc 100644
>>> --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.h
>>> +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.h
>>> @@ -26,6 +26,8 @@ enum rxe_counters {
>>>       RXE_CNT_LINK_DOWNED,
>>>       RXE_CNT_RDMA_SEND,
>>>       RXE_CNT_RDMA_RECV,
>>> +    RXE_CNT_SENT_BYTES,
>>> +    RXE_CNT_RCVD_BYTES,
>>>       RXE_NUM_OF_COUNTERS
>>>   };
>>>   diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/ 
>>> infiniband/sw/rxe/rxe_net.c
>>> index 6621d01ac32d..86660031ffa2 100644
>>> --- a/drivers/infiniband/sw/rxe/rxe_net.c
>>> +++ b/drivers/infiniband/sw/rxe/rxe_net.c
>>> @@ -503,6 +503,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct 
>>> rxe_pkt_info *pkt,
>>>       int err;
>>>       int is_request = pkt->mask & RXE_REQ_MASK;
>>>       struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
>>> +    unsigned int skblen = skb->len;
>>>       unsigned long flags;
>>>         spin_lock_irqsave(&qp->state_lock, flags);
>>> @@ -526,6 +527,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct 
>>> rxe_pkt_info *pkt,
>>>       }
>>>         rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
>>> +    rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skblen);
>>>       goto done;
>>>     drop:
>>> diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/ 
>>> infiniband/sw/rxe/rxe_recv.c
>>> index 5861e4244049..e7bab89e7d8d 100644
>>> --- a/drivers/infiniband/sw/rxe/rxe_recv.c
>>> +++ b/drivers/infiniband/sw/rxe/rxe_recv.c
>>> @@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb)
>>>       int err;
>>>       struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
>>>       struct rxe_dev *rxe = pkt->rxe;
>>> +    unsigned int skblen = skb->len - skb_network_offset(skb);
>>>         if (unlikely(skb->len < RXE_BTH_BYTES))
>>>           goto drop;
>>> @@ -341,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb)
>>>       if (unlikely(err))
>>>           goto drop;
>>>   +    rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);
>>>       rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS);
>>>         if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))
>>
>> int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
>>                     struct sk_buff *skb)
>> {
>>         int err;
>>         int is_request = pkt->mask & RXE_REQ_MASK;
>>         struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
>>         unsigned long flags;
>>
>> skb->len is printed here, that is len1
>> ...
>>         if (pkt->mask & RXE_LOOPBACK_MASK)
>>                 err = rxe_loopback(skb, pkt);
>>         else
>>                 err = rxe_send(skb, pkt);
>> ...
>> }
>>
>> In the following function
>>
>> static int rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt)
>> {
>> ...
>>         if (skb->protocol == htons(ETH_P_IP))
>>                 skb_pull(skb, sizeof(struct iphdr));
>>         else
>>                 skb_pull(skb, sizeof(struct ipv6hdr));
>>
>> ...
>>         /* remove udp header */
>>         skb_pull(skb, sizeof(struct udphdr));
>>
>> print skb->len here, that is len2
>>
>>         rxe_rcv(skb);
>>
>> ...
>> }
>>
>> Does len1 equal to len2?
> 
> I have made tests. The difference between len1 and len2 is 28.
> 
> It should be the total of ipv4 header + udp header because I use ipv4 
> address to make tests.
> 
> I am not sure if the bytes of recv data should equal to the bytes of 
> xmit data.
> 
> 
> Zhu Yanjun
> 

Yes, I got the same result of 28 bytes difference during v4 version. So 
I try to use such logic to recalculate the length:

+	unsigned int skblen = skb->len + sizeof(struct udphdr);
...
+	if (skb->protocol == htons(ETH_P_IP))
+		skblen += sizeof(struct iphdr);
+	else if (skb->protocol == htons(ETH_P_IPV6))
+		skblen += sizeof(struct ipv6hdr);
+	rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);

then use 'unsigned int skblen = skb->len - skb_network_offset(skb);' in v5.

>>
>> If not, the transmitted length appears to differ from the received 
>> length when using loopback.
>>
>> I am not sure whether this is expected behavior.
>>
>> The same observation also applies to the non-loopback case.
>>
>> Zhu Yanjun
>>
>>> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/ 
>>> infiniband/sw/rxe/rxe_verbs.h
>>> index e800545d1046..0f5ffd94643f 100644
>>> --- a/drivers/infiniband/sw/rxe/rxe_verbs.h
>>> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
>>> @@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct 
>>> rxe_dev *rxe, enum rxe_counters index)
>>>       atomic64_inc(&rxe->stats_counters[index]);
>>>   }
>>>   +static inline void rxe_counter_add(struct rxe_dev *rxe, enum 
>>> rxe_counters index,
>>> +                   s64 val)
>>> +{
>>> +    atomic64_add(val, &rxe->stats_counters[index]);
>>> +}
>>> +
>>>   static inline struct rxe_dev *to_rdev(struct ib_device *dev)
>>>   {
>>>       return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
>