Re: [PATCH net-next v3 2/4] selftests: drv-net: add an iperf performance test

[Stanislav Fomichev <sdf.kernel@gmail.com>]

On 11/03, Björn Töpel wrote:
> Stanislav Fomichev <sdf.kernel@gmail.com> writes:
> 
> > Add the drivers/net/perf target, with a lib/py of its own like the one of
> > drivers/net/hw. Add Iperf3Runner.measure(), which returns an Iperf3Result
> > that other results can join later, and report its throughput from the test.
> >
> > Single-stream iperf3 throughput is flaky and can change considerably
> > between back-to-back runs on the same hosts.
> >
> > Tested on two mlx5 hosts. Captured metric output:
> >
> >   # # ktap-metric-json: {"goal":"higher","name":"throughput","value":49.92071602072054}
> >
> > Signed-off-by: Stanislav Fomichev <sdf@fomichev.me>
> > ---
> >  tools/testing/selftests/Makefile              |  2 +-
> >  .../selftests/drivers/net/lib/py/load.py      | 20 ++++++-
> >  .../selftests/drivers/net/perf/Makefile       | 11 ++++
> >  .../testing/selftests/drivers/net/perf/config |  1 +
> >  .../selftests/drivers/net/perf/iperf.py       | 23 ++++++++
> >  .../drivers/net/perf/lib/py/__init__.py       | 57 +++++++++++++++++++
> >  .../selftests/drivers/net/perf/settings       |  1 +
> >  7 files changed, 111 insertions(+), 4 deletions(-)
> >  create mode 100644 tools/testing/selftests/drivers/net/perf/Makefile
> >  create mode 100644 tools/testing/selftests/drivers/net/perf/config
> >  create mode 100755 tools/testing/selftests/drivers/net/perf/iperf.py
> >  create mode 100644 tools/testing/selftests/drivers/net/perf/lib/py/__init__.py
> >  create mode 100644 tools/testing/selftests/drivers/net/perf/settings
> >
> > diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
> > index 07c3719f8b04..c55d7792b27c 100644
> > --- a/tools/testing/selftests/Makefile
> > +++ b/tools/testing/selftests/Makefile
> > @@ -159,7 +159,7 @@ TARGETS_HOTPLUG = cpu-hotplug
> >  TARGETS_HOTPLUG += memory-hotplug
> >  
> >  # Networking tests want the net/lib target, include it automatically
> > -ifneq ($(filter net drivers/net drivers/net/hw,$(TARGETS)),)
> > +ifneq ($(filter net drivers/net drivers/net/hw drivers/net/perf,$(TARGETS)),)
> >  ifeq ($(filter net/lib,$(TARGETS)),)
> >  	INSTALL_DEP_TARGETS := net/lib
> >  endif
> > diff --git a/tools/testing/selftests/drivers/net/lib/py/load.py b/tools/testing/selftests/drivers/net/lib/py/load.py
> > index e24660e5c27f..f337b576c4b4 100644
> > --- a/tools/testing/selftests/drivers/net/lib/py/load.py
> > +++ b/tools/testing/selftests/drivers/net/lib/py/load.py
> > @@ -3,10 +3,18 @@
> >  import re
> >  import time
> >  import json
> > +from dataclasses import dataclass
> >  
> >  from lib.py import ksft_pr, cmd, ip, rand_port, wait_port_listen
> >  
> >  
> > +@dataclass
> > +class Iperf3Result:
> > +    """Average throughput (Gbps) of an iperf3 run."""
> > +
> > +    throughput: float
> > +
> > +
> >  class Iperf3Runner:
> >      """
> >      Sets up and runs iperf3 traffic.
> > @@ -53,9 +61,9 @@ from lib.py import ksft_pr, cmd, ip, rand_port, wait_port_listen
> >              kwargs["timeout"] = duration + 5
> >          return cmd(cmdline, **kwargs)
> >  
> > -    def measure_bandwidth(self, reverse=False):
> > +    def measure(self, reverse=False):
> >          """
> > -        Runs an iperf3 measurement and returns the average bandwidth (Gbps).
> > +        Runs an iperf3 measurement and returns an Iperf3Result.
> >          Discards the first and last few reporting intervals and uses only the
> >          middle part of the run where throughput is typically stable.
> >          """
> > @@ -78,7 +86,13 @@ from lib.py import ksft_pr, cmd, ip, rand_port, wait_port_listen
> >  
> >          avg = sum(stable) / len(stable)
> >  
> > -        return avg
> > +        return Iperf3Result(avg)
> > +
> > +    def measure_bandwidth(self, reverse=False):
> > +        """
> > +        Runs an iperf3 measurement and returns the average bandwidth (Gbps).
> > +        """
> > +        return self.measure(reverse=reverse).throughput
> >  
> >  
> >  class GenerateTraffic:
> > diff --git a/tools/testing/selftests/drivers/net/perf/Makefile b/tools/testing/selftests/drivers/net/perf/Makefile
> > new file mode 100644
> > index 000000000000..31b0b60de4f6
> > --- /dev/null
> > +++ b/tools/testing/selftests/drivers/net/perf/Makefile
> > @@ -0,0 +1,11 @@
> > +# SPDX-License-Identifier: GPL-2.0
> > +
> > +TEST_PROGS := \
> > +	iperf.py \
> > +# end of TEST_PROGS
> > +
> > +TEST_INCLUDES := \
> > +	$(wildcard lib/py/*.py ../lib/py/*.py) \
> > +# end of TEST_INCLUDES
> > +
> > +include ../../../lib.mk
> > diff --git a/tools/testing/selftests/drivers/net/perf/config b/tools/testing/selftests/drivers/net/perf/config
> > new file mode 100644
> > index 000000000000..6450f34933e6
> > --- /dev/null
> > +++ b/tools/testing/selftests/drivers/net/perf/config
> > @@ -0,0 +1 @@
> > +CONFIG_IPV6=y
> > diff --git a/tools/testing/selftests/drivers/net/perf/iperf.py b/tools/testing/selftests/drivers/net/perf/iperf.py
> > new file mode 100755
> > index 000000000000..5b6f0bce2a03
> > --- /dev/null
> > +++ b/tools/testing/selftests/drivers/net/perf/iperf.py
> > @@ -0,0 +1,23 @@
> > +#!/usr/bin/env python3
> > +# SPDX-License-Identifier: GPL-2.0
> > +
> > +"""Measure TCP throughput with iperf3."""
> > +
> > +from lib.py import Iperf3Runner, NetDrvEpEnv, ksft_exit, ksft_metric, ksft_run
> > +
> > +
> > +def tcp(cfg):
> > +    """Measure TCP throughput."""
> > +    result = Iperf3Runner(cfg).measure()
> > +    ksft_metric("throughput", result.throughput, goal="higher")
> > +
> > +
> > +def main():
> > +    """Run the iperf performance test."""
> > +    with NetDrvEpEnv(__file__, nsim_test=False) as cfg:
> > +        ksft_run([tcp], args=(cfg,))
> > +    ksft_exit()
> > +
> > +
> > +if __name__ == "__main__":
> > +    main()
> > diff --git a/tools/testing/selftests/drivers/net/perf/lib/py/__init__.py b/tools/testing/selftests/drivers/net/perf/lib/py/__init__.py
> > new file mode 100644
> > index 000000000000..2bb2cd08ff7d
> > --- /dev/null
> > +++ b/tools/testing/selftests/drivers/net/perf/lib/py/__init__.py
> > @@ -0,0 +1,57 @@
> > +# SPDX-License-Identifier: GPL-2.0
> > +
> > +"""
> > +Driver test environment (performance tests).
> > +NetDrvEnv and NetDrvEpEnv are the main environment classes.
> > +NetDrvContEnv extends NetDrvEpEnv with netkit container support.
> > +Former is for local host only tests, latter creates / connects
> > +to a remote endpoint. See NIPA wiki for more information about
> > +running and writing driver tests.
> > +"""
> > +
> > +import sys
> > +from pathlib import Path
> > +
> > +KSFT_DIR = (Path(__file__).parent / "../../../../..").resolve()
> > +
> > +try:
> > +    sys.path.append(KSFT_DIR.as_posix())
> > +
> > +    # Import one by one to avoid pylint false positives
> > +    from net.lib.py import NetNS, NetNSEnter, NetdevSimDev, UserNetNS
> > +    from net.lib.py import EthtoolFamily, NetdevFamily, NetshaperFamily, \
> > +        NlError, RtnlFamily, DevlinkFamily, PSPFamily, Netlink
> > +    from net.lib.py import CmdExitFailure
> > +    from net.lib.py import bkg, cmd, bpftool, bpftrace, defer, ethtool, \
> > +        fd_read_timeout, ip, rand_port, rand_ports, wait_port_listen, \
> > +        wait_file, ctl_file_write, tool
> > +    from net.lib.py import bpf_map_set, bpf_map_dump, bpf_prog_map_ids
> > +    from net.lib.py import KsftSkipEx, KsftFailEx, KsftXfailEx
> > +    from net.lib.py import ksft_disruptive, ksft_exit, ksft_metric, ksft_pr, \
> > +        ksft_run, ksft_setup, ksft_variants, KsftNamedVariant
> > +    from net.lib.py import ksft_eq, ksft_ge, ksft_in, ksft_is, ksft_lt, \
> > +        ksft_ne, ksft_not_in, ksft_raises, ksft_true, ksft_gt, ksft_not_none
> > +    from drivers.net.lib.py import set_ethtool_feat
> > +    from drivers.net.lib.py import GenerateTraffic, Remote, Iperf3Runner
> > +    from drivers.net.lib.py import NetDrvEnv, NetDrvEpEnv, NetDrvContEnv
> > +
> > +    __all__ = ["NetNS", "NetNSEnter", "NetdevSimDev", "UserNetNS",
> > +               "EthtoolFamily", "NetdevFamily", "NetshaperFamily",
> > +               "NlError", "RtnlFamily", "DevlinkFamily", "PSPFamily", "Netlink",
> > +               "CmdExitFailure",
> > +               "bkg", "cmd", "bpftool", "bpftrace", "defer", "ethtool",
> > +               "fd_read_timeout", "ip", "rand_port", "rand_ports",
> > +               "wait_port_listen", "wait_file", "ctl_file_write", "tool",
> > +               "bpf_map_set", "bpf_map_dump", "bpf_prog_map_ids",
> > +               "KsftSkipEx", "KsftFailEx", "KsftXfailEx",
> > +               "ksft_disruptive", "ksft_exit", "ksft_metric", "ksft_pr",
> > +               "ksft_run", "ksft_setup", "ksft_variants", "KsftNamedVariant",
> > +               "ksft_eq", "ksft_ge", "ksft_in", "ksft_is", "ksft_lt",
> > +               "ksft_ne", "ksft_not_in", "ksft_raises", "ksft_true", "ksft_gt",
> > +               "ksft_not_none",
> > +               "NetDrvEnv", "NetDrvEpEnv", "NetDrvContEnv", "GenerateTraffic",
> > +               "Remote", "Iperf3Runner", "set_ethtool_feat"]
> > +except ModuleNotFoundError as e:
> > +    print("Failed importing `net` library from kernel sources")
> > +    print(str(e))
> > +    sys.exit(4)
> 
> nit: is there a KSFT_SKIP symbol?
> 
> Reviewed-by: Björn Töpel <bjorn@kernel.org>

It is a copy-paste from selftests/drivers/net/hw/lib/py/__init__.py (and there
is a similar code in net/lib/py/ynl.py). I guess I can add "# KSFT_SKIP"
here instead of doing the real import.