Re: [PATCH] mountstats: Fix per-operation percentages with nconnect

[Steve Dickson <steved@redhat.com>]

On 2/28/26 12:46 PM, Chuck Lever wrote:
> From: Chuck Lever <chuck.lever@oracle.com>
> 
> Per-operation percentages reported by "mountstats --rpc" are
> inaccurate when an NFS mount uses nconnect.
> 
> With nconnect=N, the kernel emits N separate "xprt:" lines in
> /proc/self/mountstats, one per transport.  Each transport tracks
> its own rpcsends counter reflecting only RPCs routed through that
> connection.
> 
> The parser overwrites rpcsends on each "xprt:" line, keeping only
> the last transport's value.  Per-operation counts (READ, WRITE,
> etc.) are maintained in a single array per RPC client and reflect
> all RPCs across all transports.
> 
> With nconnect=3 and balanced round-robin, rpcsends holds roughly
> one third of total RPCs while per-op counts hold the full total.
> display_rpc_op_stats() computes (op_count * 100) / rpcsends,
> yielding percentages roughly three times too large.
> 
> Accumulate rpcsends, rpcreceives, badxids, backlogutil,
> sendutil, and pendutil across multiple "xprt:" lines. These are
> cumulative counters where the sum across transports gives the
> correct aggregate.  Per-connection properties (port, bind_count,
> connect_count, connect_time, idle_time, maxslots, inflightsends)
> retain the value from the last transport seen.
> 
> Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Committed... (tag: nfs-utils-2-8-6-rc4)

steved.
> ---
>   tools/mountstats/mountstats.py | 61 +++++++++++++++++++++++++---------
>   1 file changed, 46 insertions(+), 15 deletions(-)
> 
> diff --git a/tools/mountstats/mountstats.py b/tools/mountstats/mountstats.py
> index d488f9e1c258..a6adab344d0e 100755
> --- a/tools/mountstats/mountstats.py
> +++ b/tools/mountstats/mountstats.py
> @@ -140,6 +140,38 @@ XprtRdmaCounters = [
>       'reply_waits_for_send',
>   ]
>   
> +# Counters that should be summed across transports when nconnect > 1.
> +# Each is stored in a per-transport structure in the kernel
> +# (xprt->stat or rpcrdma_xprt.rx_stats) and represents a cumulative
> +# event count or utilization value.  Per-connection properties (port,
> +# bind_count, connect_count, connect_time, idle_time, maxslots,
> +# inflightsends) retain the value from the last transport seen.
> +XprtAccumulatedCounters = {
> +    'rpcsends',
> +    'rpcreceives',
> +    'badxids',
> +    'backlogutil',
> +    'sendutil',
> +    'pendutil',
> +    'read_segments',
> +    'write_segments',
> +    'reply_segments',
> +    'total_rdma_req',
> +    'total_rdma_rep',
> +    'pullup',
> +    'fixup',
> +    'hardway',
> +    'failed_marshal',
> +    'bad_reply',
> +    'nomsg_calls',
> +    'recovered_mrs',
> +    'orphaned_mrs',
> +    'allocated_mrs',
> +    'local_invalidates',
> +    'empty_sendctx_q',
> +    'reply_waits_for_send',
> +}
> +
>   Nfsv3ops = [
>       'NULL',
>       'GETATTR',
> @@ -291,23 +323,22 @@ class DeviceData:
>           elif words[0] == 'xprt:':
>               self.__rpc_data['protocol'] = words[1]
>               if words[1] == 'udp':
> -                i = 2
> -                for key in XprtUdpCounters:
> -                    if i < len(words):
> -                        self.__rpc_data[key] = int(words[i])
> -                    i += 1
> +                counters = XprtUdpCounters
>               elif words[1] == 'tcp':
> -                i = 2
> -                for key in XprtTcpCounters:
> -                    if i < len(words):
> -                        self.__rpc_data[key] = int(words[i])
> -                    i += 1
> +                counters = XprtTcpCounters
>               elif words[1] == 'rdma':
> -                i = 2
> -                for key in XprtRdmaCounters:
> -                    if i < len(words):
> -                        self.__rpc_data[key] = int(words[i])
> -                    i += 1
> +                counters = XprtRdmaCounters
> +            else:
> +                counters = []
> +            i = 2
> +            for key in counters:
> +                if i < len(words):
> +                    val = int(words[i])
> +                    if key in XprtAccumulatedCounters and key in self.__rpc_data:
> +                        self.__rpc_data[key] += val
> +                    else:
> +                        self.__rpc_data[key] = val
> +                i += 1
>           elif words[0] == 'per-op':
>               self.__rpc_data['per-op'] = words
>           else: