* [RFC net-next v2 1/5] selftests: net: py: add timestamped metric output
2026-10-05 20:11 [RFC net-next v2 0/5] selftests: net: add performance metric reporting Stanislav Fomichev
@ 2026-10-05 20:11 ` Stanislav Fomichev
2026-10-16 16:52 ` Jakub Kicinski
2026-10-05 20:11 ` [RFC net-next v2 2/5] selftests: net: py: add metric policy output Stanislav Fomichev
` (3 subsequent siblings)
4 siblings, 1 reply; 10+ messages in thread
From: Stanislav Fomichev @ 2026-10-05 20:11 UTC (permalink / raw)
To: netdev; +Cc: davem, edumazet, kuba, pabeni
Add ksft_metric() to record timestamped JSON observations for the current
test case.
Signed-off-by: Stanislav Fomichev <sdf@fomichev.me>
---
v1: https://lore.kernel.org/netdev/20260916190409.1222272-1-sdf@fomichev.me/
---
.../testing/selftests/drivers/net/README.rst | 28 +++++++++++
.../drivers/net/hw/lib/py/__init__.py | 8 +--
.../selftests/drivers/net/lib/py/__init__.py | 8 +--
.../testing/selftests/net/lib/py/__init__.py | 9 ++--
tools/testing/selftests/net/lib/py/ksft.py | 49 ++++++++++++++++++-
5 files changed, 89 insertions(+), 13 deletions(-)
diff --git a/tools/testing/selftests/drivers/net/README.rst b/tools/testing/selftests/drivers/net/README.rst
index 3fe49bce4f3a..3e3cc028d77a 100644
--- a/tools/testing/selftests/drivers/net/README.rst
+++ b/tools/testing/selftests/drivers/net/README.rst
@@ -259,6 +259,34 @@ ksft_pr()
Use ``ksft_pr()`` instead of ``print()`` to avoid breaking TAP format.
+ksft_metric()
+~~~~~~~~~~~~~
+
+Use ``ksft_metric()`` to report a numeric measurement for the current test
+case. The helper records the time relative to the start of the case and emits
+each observation immediately before the test result as compact JSON. For
+example::
+
+ ksft_metric("throughput", 100.5, kind="scalar")
+
+emits output similar to::
+
+ # ktap-metric-json: {"name":"throughput","time":1.234,"type":"scalar","value":100.5}
+
+When ``run_kselftest.sh`` nests the test output, its diagnostic prefix makes
+this ``# # ktap-metric-json: <json>``. A background sampler can call
+``ksft_metric()`` repeatedly. Pass ``kind`` explicitly; it is emitted as the
+``type`` field. Supported kinds are ``scalar`` for one numeric value and
+``per-cpu`` for an array indexed by CPU number, with ``None`` for a missing or
+offline CPU::
+
+ ksft_metric("cpu.utilization", [75.0, 22.0, None, 100.0],
+ kind="per-cpu", host="local")
+
+The helper does not infer the kind from the Python value, since an array may
+represent something other than per-CPU data. Metrics are measurements only and
+do not affect the pass or fail result.
+
ksft_disruptive
~~~~~~~~~~~~~~~
diff --git a/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py b/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py
index 4077f5f1a1be..77e4ef34f8b9 100644
--- a/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py
+++ b/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py
@@ -27,8 +27,8 @@ KSFT_DIR = (Path(__file__).parent / "../../../../..").resolve()
wait_file, ctl_file_write, tool
from net.lib.py import bpf_map_set, bpf_map_dump, bpf_prog_map_ids
from net.lib.py import KsftSkipEx, KsftFailEx, KsftXfailEx
- from net.lib.py import ksft_disruptive, ksft_exit, ksft_pr, ksft_run, \
- ksft_setup, ksft_variants, KsftNamedVariant
+ from net.lib.py import ksft_disruptive, ksft_exit, ksft_metric, ksft_pr, \
+ ksft_run, ksft_setup, ksft_variants, KsftNamedVariant
from net.lib.py import ksft_eq, ksft_ge, ksft_in, ksft_is, ksft_lt, \
ksft_ne, ksft_not_in, ksft_raises, ksft_true, ksft_gt, ksft_not_none
from drivers.net.lib.py import set_ethtool_feat
@@ -44,8 +44,8 @@ KSFT_DIR = (Path(__file__).parent / "../../../../..").resolve()
"wait_port_listen", "wait_file", "ctl_file_write", "tool",
"bpf_map_set", "bpf_map_dump", "bpf_prog_map_ids",
"KsftSkipEx", "KsftFailEx", "KsftXfailEx",
- "ksft_disruptive", "ksft_exit", "ksft_pr", "ksft_run",
- "ksft_setup", "ksft_variants", "KsftNamedVariant",
+ "ksft_disruptive", "ksft_exit", "ksft_metric", "ksft_pr",
+ "ksft_run", "ksft_setup", "ksft_variants", "KsftNamedVariant",
"ksft_eq", "ksft_ge", "ksft_in", "ksft_is", "ksft_lt",
"ksft_ne", "ksft_not_in", "ksft_raises", "ksft_true", "ksft_gt",
"ksft_not_none", "ksft_not_none",
diff --git a/tools/testing/selftests/drivers/net/lib/py/__init__.py b/tools/testing/selftests/drivers/net/lib/py/__init__.py
index 40ccdcf0c378..f744280dcc37 100644
--- a/tools/testing/selftests/drivers/net/lib/py/__init__.py
+++ b/tools/testing/selftests/drivers/net/lib/py/__init__.py
@@ -27,8 +27,8 @@ KSFT_DIR = (Path(__file__).parent / "../../../..").resolve()
wait_file, ctl_file_write
from net.lib.py import bpf_map_set, bpf_map_dump, bpf_prog_map_ids
from net.lib.py import KsftSkipEx, KsftFailEx, KsftXfailEx
- from net.lib.py import ksft_disruptive, ksft_exit, ksft_pr, ksft_run, \
- ksft_setup, ksft_variants, KsftNamedVariant
+ from net.lib.py import ksft_disruptive, ksft_exit, ksft_metric, ksft_pr, \
+ ksft_run, ksft_setup, ksft_variants, KsftNamedVariant
from net.lib.py import ksft_eq, ksft_ge, ksft_in, ksft_is, ksft_lt, \
ksft_ne, ksft_not_in, ksft_raises, ksft_true, ksft_gt, ksft_not_none
@@ -41,8 +41,8 @@ KSFT_DIR = (Path(__file__).parent / "../../../..").resolve()
"wait_port_listen", "wait_file", "ctl_file_write",
"bpf_map_set", "bpf_map_dump", "bpf_prog_map_ids",
"KsftSkipEx", "KsftFailEx", "KsftXfailEx",
- "ksft_disruptive", "ksft_exit", "ksft_pr", "ksft_run",
- "ksft_setup", "ksft_variants", "KsftNamedVariant",
+ "ksft_disruptive", "ksft_exit", "ksft_metric", "ksft_pr",
+ "ksft_run", "ksft_setup", "ksft_variants", "KsftNamedVariant",
"ksft_eq", "ksft_ge", "ksft_in", "ksft_is", "ksft_lt",
"ksft_ne", "ksft_not_in", "ksft_raises", "ksft_true", "ksft_gt",
"ksft_not_none", "ksft_not_none"]
diff --git a/tools/testing/selftests/net/lib/py/__init__.py b/tools/testing/selftests/net/lib/py/__init__.py
index 71df5880b356..3d9e6f6b402c 100644
--- a/tools/testing/selftests/net/lib/py/__init__.py
+++ b/tools/testing/selftests/net/lib/py/__init__.py
@@ -8,8 +8,8 @@ from .consts import KSRC
from .ksft import KsftFailEx, KsftSkipEx, KsftXfailEx, ksft_pr, ksft_eq, \
ksft_ne, ksft_true, ksft_not_none, ksft_in, ksft_not_in, ksft_is, \
ksft_ge, ksft_gt, ksft_lt, ksft_raises, ksft_busy_wait, \
- ktap_result, ksft_disruptive, ksft_setup, ksft_run, ksft_exit, \
- ksft_variants, KsftNamedVariant
+ ktap_result, ksft_disruptive, ksft_metric, ksft_setup, ksft_run, \
+ ksft_exit, ksft_variants, KsftNamedVariant
from .netns import NetNS, NetNSEnter, UserNetNS
from .nsim import NetdevSim, NetdevSimDev
from .utils import CmdExitFailure, fd_read_timeout, cmd, bkg, defer, \
@@ -24,8 +24,9 @@ __all__ = ["KSRC",
"KsftFailEx", "KsftSkipEx", "KsftXfailEx", "ksft_pr", "ksft_eq",
"ksft_ne", "ksft_true", "ksft_not_none", "ksft_in", "ksft_not_in",
"ksft_is", "ksft_ge", "ksft_gt", "ksft_lt", "ksft_raises",
- "ksft_busy_wait", "ktap_result", "ksft_disruptive", "ksft_setup",
- "ksft_run", "ksft_exit", "ksft_variants", "KsftNamedVariant",
+ "ksft_busy_wait", "ktap_result", "ksft_disruptive", "ksft_metric",
+ "ksft_setup", "ksft_run", "ksft_exit", "ksft_variants",
+ "KsftNamedVariant",
"NetNS", "NetNSEnter", "UserNetNS",
"CmdExitFailure", "fd_read_timeout", "cmd", "bkg", "defer",
"bpftool", "ip", "ethtool", "bpftrace", "rand_port", "rand_ports",
diff --git a/tools/testing/selftests/net/lib/py/ksft.py b/tools/testing/selftests/net/lib/py/ksft.py
index 81287c2daff0..1259bbb6d6c5 100644
--- a/tools/testing/selftests/net/lib/py/ksft.py
+++ b/tools/testing/selftests/net/lib/py/ksft.py
@@ -4,9 +4,11 @@ import fnmatch
import functools
import getopt
import inspect
+import json
import os
import signal
import sys
+import threading
import time
import traceback
from collections import namedtuple
@@ -16,6 +18,9 @@ from . import utils
KSFT_RESULT = None
KSFT_RESULT_ALL = True
KSFT_DISRUPTIVE = True
+KSFT_METRICS = None
+KSFT_METRICS_START = None
+KSFT_METRICS_LOCK = threading.Lock()
class KsftFailEx(Exception):
@@ -93,6 +98,44 @@ KSFT_DISRUPTIVE = True
print(pfx, prefixed, **kwargs)
+def ksft_metric(name, value, *, kind, **labels):
+ """Record a timestamped metric with an explicitly described kind.
+
+ The kind is emitted as the ``type`` field. Supported kinds are ``scalar``
+ for one numeric value and ``per-cpu`` for an array indexed by CPU number,
+ with ``None`` for a missing or offline CPU.
+ """
+ with KSFT_METRICS_LOCK:
+ if KSFT_METRICS is None or KSFT_METRICS_START is None:
+ raise RuntimeError("ksft_metric() called outside of a test case")
+
+ metric = {
+ "name": name,
+ "type": kind,
+ "value": value,
+ "time": round(time.monotonic() - KSFT_METRICS_START, 6),
+ }
+ for key, label_value in sorted(labels.items()):
+ if key in metric:
+ raise ValueError(f"Metric label uses reserved name: {key}")
+ metric[key] = label_value
+ KSFT_METRICS.append(metric)
+
+
+def _ksft_flush_metrics():
+ global KSFT_METRICS, KSFT_METRICS_START
+
+ with KSFT_METRICS_LOCK:
+ metrics = KSFT_METRICS
+ KSFT_METRICS = None
+ KSFT_METRICS_START = None
+
+ for metric in metrics or []:
+ payload = json.dumps(metric, allow_nan=False, separators=(",", ":"),
+ sort_keys=True)
+ ksft_pr(f"ktap-metric-json: {payload}")
+
+
def _fail(*args):
global KSFT_RESULT
KSFT_RESULT = False
@@ -400,7 +443,7 @@ KsftCaseFunction = namedtuple("KsftCaseFunction",
totals = {"pass": 0, "fail": 0, "skip": 0, "xfail": 0}
- global KSFT_RESULT
+ global KSFT_RESULT, KSFT_METRICS, KSFT_METRICS_START
if KSFT_RESULT is not None:
raise RuntimeError("ksft_run() can't be called multiple times.")
@@ -411,6 +454,9 @@ KsftCaseFunction = namedtuple("KsftCaseFunction",
stop = False
for func, args, name in test_cases:
KSFT_RESULT = True
+ with KSFT_METRICS_LOCK:
+ KSFT_METRICS = []
+ KSFT_METRICS_START = time.monotonic()
cnt += 1
comment = ""
cnt_key = ""
@@ -450,6 +496,7 @@ KsftCaseFunction = namedtuple("KsftCaseFunction",
if not cnt_key:
cnt_key = 'pass' if KSFT_RESULT else 'fail'
+ _ksft_flush_metrics()
ktap_result(KSFT_RESULT, cnt, name, comment=comment)
totals[cnt_key] += 1
--
2.53.0-Meta
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [RFC net-next v2 1/5] selftests: net: py: add timestamped metric output
2026-10-05 20:11 ` [RFC net-next v2 1/5] selftests: net: py: add timestamped metric output Stanislav Fomichev
@ 2026-10-16 16:52 ` Jakub Kicinski
0 siblings, 0 replies; 10+ messages in thread
From: Jakub Kicinski @ 2026-10-16 16:52 UTC (permalink / raw)
To: Stanislav Fomichev; +Cc: netdev, davem, edumazet, pabeni
On Mon, 5 Oct 2026 13:11:10 -0700 Stanislav Fomichev wrote:
> +ksft_metric()
> +~~~~~~~~~~~~~
> +
> +Use ``ksft_metric()`` to report a numeric measurement for the current test
> +case. The helper records the time relative to the start of the case and emits
> +each observation immediately before the test result as compact JSON. For
> +example::
> +
> + ksft_metric("throughput", 100.5, kind="scalar")
> +
> +emits output similar to::
> +
> + # ktap-metric-json: {"name":"throughput","time":1.234,"type":"scalar","value":100.5}
Should time be optional? In case some test wants to report single metric
not value over time? Maybe just mention that it's an option if someone
needs it in the future?
> +When ``run_kselftest.sh`` nests the test output, its diagnostic prefix makes
> +this ``# # ktap-metric-json: <json>``.
This sentence needs to go, it's out of place and obvious to anyone who
worked with ksft.
> A background sampler can call
> +``ksft_metric()`` repeatedly. Pass ``kind`` explicitly; it is emitted as the
> +``type`` field. Supported kinds are ``scalar`` for one numeric value and
> +``per-cpu`` for an array indexed by CPU number, with ``None`` for a missing or
> +offline CPU::
> +
> + ksft_metric("cpu.utilization", [75.0, 22.0, None, 100.0],
> + kind="per-cpu", host="local")
Should we say something about what the values for "kind" are?
Or should this be internal - IOW create ksft_metric_X for supported
types, like ksft_metric_scalar, which would be more self-documenting?
> +The helper does not infer the kind from the Python value, since an array may
> +represent something other than per-CPU data. Metrics are measurements only and
> +do not affect the pass or fail result.
^ permalink raw reply [flat|nested] 10+ messages in thread
* [RFC net-next v2 2/5] selftests: net: py: add metric policy output
2026-10-05 20:11 [RFC net-next v2 0/5] selftests: net: add performance metric reporting Stanislav Fomichev
2026-10-05 20:11 ` [RFC net-next v2 1/5] selftests: net: py: add timestamped metric output Stanislav Fomichev
@ 2026-10-05 20:11 ` Stanislav Fomichev
2026-10-16 16:59 ` Jakub Kicinski
2026-10-05 20:11 ` [RFC net-next v2 3/5] selftests: drv-net: add an iperf performance test Stanislav Fomichev
` (2 subsequent siblings)
4 siblings, 1 reply; 10+ messages in thread
From: Stanislav Fomichev @ 2026-10-05 20:11 UTC (permalink / raw)
To: netdev; +Cc: davem, edumazet, kuba, pabeni
Add ksft_metric_policy() so producers can describe metric processing,
display, and regression policy without name-specific consumer logic.
Signed-off-by: Stanislav Fomichev <sdf@fomichev.me>
---
.../testing/selftests/drivers/net/README.rst | 21 ++++
.../drivers/net/hw/lib/py/__init__.py | 11 +-
.../selftests/drivers/net/lib/py/__init__.py | 11 +-
.../testing/selftests/net/lib/py/__init__.py | 6 +-
tools/testing/selftests/net/lib/py/ksft.py | 114 +++++++++++++++++-
5 files changed, 151 insertions(+), 12 deletions(-)
diff --git a/tools/testing/selftests/drivers/net/README.rst b/tools/testing/selftests/drivers/net/README.rst
index 3e3cc028d77a..1b7264d40b79 100644
--- a/tools/testing/selftests/drivers/net/README.rst
+++ b/tools/testing/selftests/drivers/net/README.rst
@@ -287,6 +287,27 @@ The helper does not infer the kind from the Python value, since an array may
represent something other than per-CPU data. Metrics are measurements only and
do not affect the pass or fail result.
+ksft_metric_policy()
+~~~~~~~~~~~~~~~~~~~~
+
+Use ``ksft_metric_policy()`` before the first matching observation to
+register processing, display, and regression policy for a metric. The policy
+is emitted once per test case rather than repeated in every observation. For
+example::
+
+ ksft_metric_policy("nic.rx.dropped", summarize="max",
+ transform="rate", display_range={"min": 0})
+ ksft_metric("nic.rx.dropped", 0, kind="scalar", host="local")
+
+emits separate policy and observation records::
+
+ # ktap-metric-policy-json: {"display_range":{"min":0},"name":"nic.rx.dropped","summarize":"max","transform":"rate"}
+ # ktap-metric-json: {"host":"local","name":"nic.rx.dropped","time":1.234,"type":"scalar","value":0}
+
+Consumers apply a policy to observations with the same metric name. The
+function docstring documents the supported summary, transform, aggregation,
+regression, and display options.
+
ksft_disruptive
~~~~~~~~~~~~~~~
diff --git a/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py b/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py
index 77e4ef34f8b9..cd771c00c074 100644
--- a/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py
+++ b/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py
@@ -27,8 +27,9 @@ KSFT_DIR = (Path(__file__).parent / "../../../../..").resolve()
wait_file, ctl_file_write, tool
from net.lib.py import bpf_map_set, bpf_map_dump, bpf_prog_map_ids
from net.lib.py import KsftSkipEx, KsftFailEx, KsftXfailEx
- from net.lib.py import ksft_disruptive, ksft_exit, ksft_metric, ksft_pr, \
- ksft_run, ksft_setup, ksft_variants, KsftNamedVariant
+ from net.lib.py import ksft_disruptive, ksft_exit, ksft_metric, \
+ ksft_metric_policy, ksft_pr, ksft_run, ksft_setup, \
+ ksft_variants, KsftNamedVariant
from net.lib.py import ksft_eq, ksft_ge, ksft_in, ksft_is, ksft_lt, \
ksft_ne, ksft_not_in, ksft_raises, ksft_true, ksft_gt, ksft_not_none
from drivers.net.lib.py import set_ethtool_feat
@@ -44,8 +45,10 @@ KSFT_DIR = (Path(__file__).parent / "../../../../..").resolve()
"wait_port_listen", "wait_file", "ctl_file_write", "tool",
"bpf_map_set", "bpf_map_dump", "bpf_prog_map_ids",
"KsftSkipEx", "KsftFailEx", "KsftXfailEx",
- "ksft_disruptive", "ksft_exit", "ksft_metric", "ksft_pr",
- "ksft_run", "ksft_setup", "ksft_variants", "KsftNamedVariant",
+ "ksft_disruptive", "ksft_exit", "ksft_metric",
+ "ksft_metric_policy", "ksft_pr", "ksft_run",
+ "ksft_setup", "ksft_variants",
+ "KsftNamedVariant",
"ksft_eq", "ksft_ge", "ksft_in", "ksft_is", "ksft_lt",
"ksft_ne", "ksft_not_in", "ksft_raises", "ksft_true", "ksft_gt",
"ksft_not_none", "ksft_not_none",
diff --git a/tools/testing/selftests/drivers/net/lib/py/__init__.py b/tools/testing/selftests/drivers/net/lib/py/__init__.py
index f744280dcc37..58c2877e7928 100644
--- a/tools/testing/selftests/drivers/net/lib/py/__init__.py
+++ b/tools/testing/selftests/drivers/net/lib/py/__init__.py
@@ -27,8 +27,9 @@ KSFT_DIR = (Path(__file__).parent / "../../../..").resolve()
wait_file, ctl_file_write
from net.lib.py import bpf_map_set, bpf_map_dump, bpf_prog_map_ids
from net.lib.py import KsftSkipEx, KsftFailEx, KsftXfailEx
- from net.lib.py import ksft_disruptive, ksft_exit, ksft_metric, ksft_pr, \
- ksft_run, ksft_setup, ksft_variants, KsftNamedVariant
+ from net.lib.py import ksft_disruptive, ksft_exit, ksft_metric, \
+ ksft_metric_policy, ksft_pr, ksft_run, ksft_setup, \
+ ksft_variants, KsftNamedVariant
from net.lib.py import ksft_eq, ksft_ge, ksft_in, ksft_is, ksft_lt, \
ksft_ne, ksft_not_in, ksft_raises, ksft_true, ksft_gt, ksft_not_none
@@ -41,8 +42,10 @@ KSFT_DIR = (Path(__file__).parent / "../../../..").resolve()
"wait_port_listen", "wait_file", "ctl_file_write",
"bpf_map_set", "bpf_map_dump", "bpf_prog_map_ids",
"KsftSkipEx", "KsftFailEx", "KsftXfailEx",
- "ksft_disruptive", "ksft_exit", "ksft_metric", "ksft_pr",
- "ksft_run", "ksft_setup", "ksft_variants", "KsftNamedVariant",
+ "ksft_disruptive", "ksft_exit", "ksft_metric",
+ "ksft_metric_policy", "ksft_pr", "ksft_run",
+ "ksft_setup", "ksft_variants",
+ "KsftNamedVariant",
"ksft_eq", "ksft_ge", "ksft_in", "ksft_is", "ksft_lt",
"ksft_ne", "ksft_not_in", "ksft_raises", "ksft_true", "ksft_gt",
"ksft_not_none", "ksft_not_none"]
diff --git a/tools/testing/selftests/net/lib/py/__init__.py b/tools/testing/selftests/net/lib/py/__init__.py
index 3d9e6f6b402c..db25602e4d57 100644
--- a/tools/testing/selftests/net/lib/py/__init__.py
+++ b/tools/testing/selftests/net/lib/py/__init__.py
@@ -8,7 +8,8 @@ from .consts import KSRC
from .ksft import KsftFailEx, KsftSkipEx, KsftXfailEx, ksft_pr, ksft_eq, \
ksft_ne, ksft_true, ksft_not_none, ksft_in, ksft_not_in, ksft_is, \
ksft_ge, ksft_gt, ksft_lt, ksft_raises, ksft_busy_wait, \
- ktap_result, ksft_disruptive, ksft_metric, ksft_setup, ksft_run, \
+ ktap_result, ksft_disruptive, ksft_metric, ksft_metric_policy, \
+ ksft_setup, ksft_run, \
ksft_exit, ksft_variants, KsftNamedVariant
from .netns import NetNS, NetNSEnter, UserNetNS
from .nsim import NetdevSim, NetdevSimDev
@@ -25,7 +26,8 @@ __all__ = ["KSRC",
"ksft_ne", "ksft_true", "ksft_not_none", "ksft_in", "ksft_not_in",
"ksft_is", "ksft_ge", "ksft_gt", "ksft_lt", "ksft_raises",
"ksft_busy_wait", "ktap_result", "ksft_disruptive", "ksft_metric",
- "ksft_setup", "ksft_run", "ksft_exit", "ksft_variants",
+ "ksft_metric_policy", "ksft_setup", "ksft_run",
+ "ksft_exit", "ksft_variants",
"KsftNamedVariant",
"NetNS", "NetNSEnter", "UserNetNS",
"CmdExitFailure", "fd_read_timeout", "cmd", "bkg", "defer",
diff --git a/tools/testing/selftests/net/lib/py/ksft.py b/tools/testing/selftests/net/lib/py/ksft.py
index 1259bbb6d6c5..79cbdb6d98fe 100644
--- a/tools/testing/selftests/net/lib/py/ksft.py
+++ b/tools/testing/selftests/net/lib/py/ksft.py
@@ -20,6 +20,7 @@ KSFT_RESULT_ALL = True
KSFT_DISRUPTIVE = True
KSFT_METRICS = None
KSFT_METRICS_START = None
+KSFT_METRIC_POLICIES = None
KSFT_METRICS_LOCK = threading.Lock()
@@ -98,6 +99,104 @@ KSFT_METRICS_LOCK = threading.Lock()
print(pfx, prefixed, **kwargs)
+def ksft_metric_policy(name, summarize="total", *, aggregation=None,
+ transform=None, regression=None,
+ display_range=None, display_scale=None,
+ display_label=None):
+ """Register processing and presentation metadata for a metric.
+
+ The policy applies to every observation with ``name`` in the current test
+ case. It is emitted once as a ``ktap-metric-policy-json`` record before
+ the matching ``ktap-metric-json`` observations, so a consumer can process
+ the metric without recognizing its name. Register a policy before the
+ first matching :func:`ksft_metric` call. Registering the same policy more
+ than once is allowed; changing it or registering it after an observation
+ is rejected.
+
+ Args:
+ name: Metric name passed to :func:`ksft_metric`.
+ transform: Optional operation applied before aggregation and summary.
+ ``rate`` calculates ``(current - previous) / elapsed_time`` from
+ timestamped cumulative observations. Counter decreases and
+ intervals with nonpositive elapsed time do not produce a rate.
+ aggregation: Optional operation applied to each transformed
+ observation before summarization. An aggregation may reduce a
+ structured value, such as an array, to a scalar.
+ ``busy-core-equivalents`` expects a CPU-indexed array and sums
+ transformed per-CPU rates. Without a transform, it treats values
+ as percentages and divides their sum by 100. The resulting series
+ measures concurrently busy cores.
+ summarize: Operation used to reduce the processed observations.
+ Supported values are ``last``, ``total``, ``max``, and ``pXX``,
+ where ``XX`` is the requested percentile. The default is
+ ``total``.
+ regression: Optional regression-tracking policy dictionary:
+
+ ``compare``
+ Summary operation to compare: ``last``, ``total``, ``max``,
+ or ``pXX``, where ``XX`` is the requested percentile. This is
+ independent of the summary selected for display.
+ ``better``
+ ``higher`` or ``lower``, indicating which direction is an
+ improvement.
+ ``relative_tolerance``
+ Optional nonnegative fractional deterioration, for example
+ ``0.05`` for five percent.
+ ``absolute_tolerance``
+ Optional nonnegative deterioration in the processed metric's
+ native units. At least one tolerance is required; consumers
+ use the larger allowance when both are present.
+
+ The producer describes comparison semantics, while the consumer
+ selects historical baselines and determines regression status.
+ display_range: Optional graph range dictionary containing ``min``,
+ ``max``, or both. It describes displayed individual series. An
+ upper bound does not cap a post-aggregation total such as multiple
+ busy-core equivalents.
+ display_scale: Optional positive multiplier applied only when values
+ are displayed, after transformation. It does not alter stored
+ observations, aggregation, summaries, or regression comparison.
+ display_label: Optional nonempty axis label for displayed values.
+
+ Example::
+
+ ksft_metric_policy(
+ "cpu.time.usr", summarize="p90", transform="rate",
+ aggregation="busy-core-equivalents",
+ display_range={"min": 0, "max": 100},
+ display_scale=100,
+ display_label="Percent of one CPU")
+
+ Raises:
+ RuntimeError: If called outside a test case or after a matching metric.
+ ValueError: If the policy conflicts with an earlier registration.
+ """
+ metadata = {"summarize": summarize}
+ optional = {
+ "aggregation": aggregation,
+ "transform": transform,
+ "regression": regression,
+ "display_range": display_range,
+ "display_scale": display_scale,
+ "display_label": display_label,
+ }
+ metadata.update({key: value for key, value in optional.items()
+ if value is not None})
+
+ with KSFT_METRICS_LOCK:
+ if KSFT_METRIC_POLICIES is None:
+ raise RuntimeError(
+ "ksft_metric_policy() called outside of a test case")
+ previous = KSFT_METRIC_POLICIES.get(name)
+ if previous is not None and previous != metadata:
+ raise ValueError(f"Conflicting policy for metric {name}")
+ if previous is None and any(metric["name"] == name
+ for metric in KSFT_METRICS):
+ raise RuntimeError(
+ f"Policy registered after metric {name} was recorded")
+ KSFT_METRIC_POLICIES[name] = metadata
+
+
def ksft_metric(name, value, *, kind, **labels):
"""Record a timestamped metric with an explicitly described kind.
@@ -123,13 +222,23 @@ KSFT_METRICS_LOCK = threading.Lock()
def _ksft_flush_metrics():
- global KSFT_METRICS, KSFT_METRICS_START
+ global KSFT_METRICS, KSFT_METRICS_START, KSFT_METRIC_POLICIES
with KSFT_METRICS_LOCK:
metrics = KSFT_METRICS
+ policies = KSFT_METRIC_POLICIES
KSFT_METRICS = None
KSFT_METRICS_START = None
+ KSFT_METRIC_POLICIES = None
+ metric_names = {metric["name"] for metric in metrics or []}
+ for name, metadata in (policies or {}).items():
+ if name not in metric_names:
+ continue
+ policy = {"name": name, **metadata}
+ payload = json.dumps(policy, allow_nan=False, separators=(",", ":"),
+ sort_keys=True)
+ ksft_pr(f"ktap-metric-policy-json: {payload}")
for metric in metrics or []:
payload = json.dumps(metric, allow_nan=False, separators=(",", ":"),
sort_keys=True)
@@ -443,7 +552,7 @@ KsftCaseFunction = namedtuple("KsftCaseFunction",
totals = {"pass": 0, "fail": 0, "skip": 0, "xfail": 0}
- global KSFT_RESULT, KSFT_METRICS, KSFT_METRICS_START
+ global KSFT_RESULT, KSFT_METRICS, KSFT_METRICS_START, KSFT_METRIC_POLICIES
if KSFT_RESULT is not None:
raise RuntimeError("ksft_run() can't be called multiple times.")
@@ -457,6 +566,7 @@ KsftCaseFunction = namedtuple("KsftCaseFunction",
with KSFT_METRICS_LOCK:
KSFT_METRICS = []
KSFT_METRICS_START = time.monotonic()
+ KSFT_METRIC_POLICIES = {}
cnt += 1
comment = ""
cnt_key = ""
--
2.53.0-Meta
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [RFC net-next v2 2/5] selftests: net: py: add metric policy output
2026-10-05 20:11 ` [RFC net-next v2 2/5] selftests: net: py: add metric policy output Stanislav Fomichev
@ 2026-10-16 16:59 ` Jakub Kicinski
0 siblings, 0 replies; 10+ messages in thread
From: Jakub Kicinski @ 2026-10-16 16:59 UTC (permalink / raw)
To: Stanislav Fomichev; +Cc: netdev, davem, edumazet, pabeni
On Mon, 5 Oct 2026 13:11:11 -0700 Stanislav Fomichev wrote:
> + ksft_metric_policy("nic.rx.dropped", summarize="max",
> + transform="rate", display_range={"min": 0})
IMO we should not be mixing any of this into the test.
We can hard code some heuristics into UI to match on names and create
sensible defaults. Here this feels out of place.
I would have expected only what you have as "regression policy"
in here. And only a subset of that. Naively we want to know
- if the metric is an error metric or perf metric (error metric
must be at 0, perf metric has to be analyzed for expected perf)
- whether it's a badness or goodness metric to distinguish wins
from regressions
^ permalink raw reply [flat|nested] 10+ messages in thread
* [RFC net-next v2 3/5] selftests: drv-net: add an iperf performance test
2026-10-05 20:11 [RFC net-next v2 0/5] selftests: net: add performance metric reporting Stanislav Fomichev
2026-10-05 20:11 ` [RFC net-next v2 1/5] selftests: net: py: add timestamped metric output Stanislav Fomichev
2026-10-05 20:11 ` [RFC net-next v2 2/5] selftests: net: py: add metric policy output Stanislav Fomichev
@ 2026-10-05 20:11 ` Stanislav Fomichev
2026-10-16 17:03 ` Jakub Kicinski
2026-10-05 20:11 ` [RFC net-next v2 4/5] selftests: drv-net: add a kperf runner Stanislav Fomichev
2026-10-05 20:11 ` [RFC net-next v2 5/5] selftests: drv-net: measure devmem performance with kperf Stanislav Fomichev
4 siblings, 1 reply; 10+ messages in thread
From: Stanislav Fomichev @ 2026-10-05 20:11 UTC (permalink / raw)
To: netdev; +Cc: davem, edumazet, kuba, pabeni
Add the drivers/net/hw/performance target. Report throughput and
retransmits from Iperf3Runner when requested.
Single-stream iperf3 throughput is flaky and can change considerably
between back-to-back runs on the same hosts.
Tested on two mlx5 hosts. Captured metric output:
# # ktap-metric-policy-json: {"display_label":"Gbps","name":"throughput","regression":{"better":"higher","compare":"p50","relative_tolerance":0.05},"summarize":"p50"}
# # ktap-metric-policy-json: {"name":"tcp.retransmits","summarize":"max"}
# # ktap-metric-json: {"host":"local","name":"throughput","time":10.685632,"type":"scalar","value":48.561920255183196}
# # ktap-metric-json: {"host":"remote","name":"tcp.retransmits","time":10.68565,"type":"scalar","value":0}
Signed-off-by: Stanislav Fomichev <sdf@fomichev.me>
---
tools/testing/selftests/Makefile | 2 +-
.../drivers/net/hw/performance/Makefile | 10 +++++++
.../drivers/net/hw/performance/config | 1 +
.../drivers/net/hw/performance/iperf.py | 27 +++++++++++++++++++
.../drivers/net/hw/performance/settings | 1 +
.../selftests/drivers/net/lib/py/load.py | 24 +++++++++++++++--
6 files changed, 62 insertions(+), 3 deletions(-)
create mode 100644 tools/testing/selftests/drivers/net/hw/performance/Makefile
create mode 100644 tools/testing/selftests/drivers/net/hw/performance/config
create mode 100755 tools/testing/selftests/drivers/net/hw/performance/iperf.py
create mode 100644 tools/testing/selftests/drivers/net/hw/performance/settings
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index 273853937c25..7523f378e026 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -152,7 +152,7 @@ TARGETS_HOTPLUG = cpu-hotplug
TARGETS_HOTPLUG += memory-hotplug
# Networking tests want the net/lib target, include it automatically
-ifneq ($(filter net drivers/net drivers/net/hw,$(TARGETS)),)
+ifneq ($(filter net drivers/net drivers/net/hw drivers/net/hw/performance,$(TARGETS)),)
ifeq ($(filter net/lib,$(TARGETS)),)
INSTALL_DEP_TARGETS := net/lib
endif
diff --git a/tools/testing/selftests/drivers/net/hw/performance/Makefile b/tools/testing/selftests/drivers/net/hw/performance/Makefile
new file mode 100644
index 000000000000..5d53ef548306
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/performance/Makefile
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: GPL-2.0
+
+TEST_PROGS := iperf.py
+
+TEST_INCLUDES := \
+ $(wildcard ../lib/py/*.py ../../lib/py/*.py) \
+ $(wildcard ../../../../net/lib/py/*.py) \
+ #
+
+include ../../../../lib.mk
diff --git a/tools/testing/selftests/drivers/net/hw/performance/config b/tools/testing/selftests/drivers/net/hw/performance/config
new file mode 100644
index 000000000000..6450f34933e6
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/performance/config
@@ -0,0 +1 @@
+CONFIG_IPV6=y
diff --git a/tools/testing/selftests/drivers/net/hw/performance/iperf.py b/tools/testing/selftests/drivers/net/hw/performance/iperf.py
new file mode 100755
index 000000000000..415fb67f4100
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/performance/iperf.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+
+"""Measure TCP throughput with iperf3."""
+
+from os import path
+import sys
+
+sys.path.insert(0, path.join(path.dirname(path.abspath(__file__)), ".."))
+
+from lib.py import Iperf3Runner, NetDrvEpEnv, ksft_exit, ksft_run
+
+
+def tcp(cfg):
+ """Measure TCP throughput."""
+ Iperf3Runner(cfg).measure_bandwidth(report=True)
+
+
+def main():
+ """Run the iperf performance test."""
+ with NetDrvEpEnv(__file__, nsim_test=False) as cfg:
+ ksft_run([tcp], args=(cfg,))
+ ksft_exit()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/testing/selftests/drivers/net/hw/performance/settings b/tools/testing/selftests/drivers/net/hw/performance/settings
new file mode 100644
index 000000000000..e7b9417537fb
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/performance/settings
@@ -0,0 +1 @@
+timeout=0
diff --git a/tools/testing/selftests/drivers/net/lib/py/load.py b/tools/testing/selftests/drivers/net/lib/py/load.py
index e24660e5c27f..a7da025b4e21 100644
--- a/tools/testing/selftests/drivers/net/lib/py/load.py
+++ b/tools/testing/selftests/drivers/net/lib/py/load.py
@@ -4,7 +4,8 @@ import re
import time
import json
-from lib.py import ksft_pr, cmd, ip, rand_port, wait_port_listen
+from lib.py import cmd, ip, ksft_metric, ksft_metric_policy, ksft_pr, \
+ rand_port, wait_port_listen
class Iperf3Runner:
@@ -53,12 +54,20 @@ from lib.py import ksft_pr, cmd, ip, rand_port, wait_port_listen
kwargs["timeout"] = duration + 5
return cmd(cmdline, **kwargs)
- def measure_bandwidth(self, reverse=False):
+ def measure_bandwidth(self, reverse=False, report=False):
"""
Runs an iperf3 measurement and returns the average bandwidth (Gbps).
Discards the first and last few reporting intervals and uses only the
middle part of the run where throughput is typically stable.
"""
+ if report:
+ ksft_metric_policy(
+ "throughput", summarize="p50",
+ regression={"compare": "p50", "better": "higher",
+ "relative_tolerance": 0.05},
+ display_label="Gbps")
+ ksft_metric_policy("tcp.retransmits", summarize="max")
+
self.start_server()
result = self.start_client(duration=10, reverse=reverse)
@@ -77,6 +86,17 @@ from lib.py import ksft_pr, cmd, ip, rand_port, wait_port_listen
stable = samples[3:-3]
avg = sum(stable) / len(stable)
+ retransmits = out.get("end", {}).get("sum_sent", {}).get(
+ "retransmits")
+ if report:
+ if not isinstance(retransmits, int) or \
+ isinstance(retransmits, bool):
+ raise ValueError("iperf3 did not report TCP retransmits")
+ receiver = "remote" if reverse else "local"
+ sender = "local" if reverse else "remote"
+ ksft_metric("throughput", avg, kind="scalar", host=receiver)
+ ksft_metric("tcp.retransmits", retransmits, kind="scalar",
+ host=sender)
return avg
--
2.53.0-Meta
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [RFC net-next v2 3/5] selftests: drv-net: add an iperf performance test
2026-10-05 20:11 ` [RFC net-next v2 3/5] selftests: drv-net: add an iperf performance test Stanislav Fomichev
@ 2026-10-16 17:03 ` Jakub Kicinski
0 siblings, 0 replies; 10+ messages in thread
From: Jakub Kicinski @ 2026-10-16 17:03 UTC (permalink / raw)
To: Stanislav Fomichev; +Cc: netdev, davem, edumazet, pabeni
On Mon, 5 Oct 2026 13:11:12 -0700 Stanislav Fomichev wrote:
> Add the drivers/net/hw/performance target. Report throughput and
> retransmits from Iperf3Runner when requested.
>
> Single-stream iperf3 throughput is flaky and can change considerably
> between back-to-back runs on the same hosts.
>
> Tested on two mlx5 hosts. Captured metric output:
>
> # # ktap-metric-policy-json: {"display_label":"Gbps","name":"throughput","regression":{"better":"higher","compare":"p50","relative_tolerance":0.05},"summarize":"p50"}
> # # ktap-metric-policy-json: {"name":"tcp.retransmits","summarize":"max"}
> # # ktap-metric-json: {"host":"local","name":"throughput","time":10.685632,"type":"scalar","value":48.561920255183196}
> # # ktap-metric-json: {"host":"remote","name":"tcp.retransmits","time":10.68565,"type":"scalar","value":0}
>
> Signed-off-by: Stanislav Fomichev <sdf@fomichev.me>
> ---
> tools/testing/selftests/Makefile | 2 +-
> .../drivers/net/hw/performance/Makefile | 10 +++++++
> .../drivers/net/hw/performance/config | 1 +
> .../drivers/net/hw/performance/iperf.py | 27 +++++++++++++++++++
> .../drivers/net/hw/performance/settings | 1 +
> .../selftests/drivers/net/lib/py/load.py | 24 +++++++++++++++--
> 6 files changed, 62 insertions(+), 3 deletions(-)
> create mode 100644 tools/testing/selftests/drivers/net/hw/performance/Makefile
> create mode 100644 tools/testing/selftests/drivers/net/hw/performance/config
> create mode 100755 tools/testing/selftests/drivers/net/hw/performance/iperf.py
> create mode 100644 tools/testing/selftests/drivers/net/hw/performance/settings
>
> diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
> index 273853937c25..7523f378e026 100644
> --- a/tools/testing/selftests/Makefile
> +++ b/tools/testing/selftests/Makefile
> @@ -152,7 +152,7 @@ TARGETS_HOTPLUG = cpu-hotplug
> TARGETS_HOTPLUG += memory-hotplug
>
> # Networking tests want the net/lib target, include it automatically
> -ifneq ($(filter net drivers/net drivers/net/hw,$(TARGETS)),)
> +ifneq ($(filter net drivers/net drivers/net/hw drivers/net/hw/performance,$(TARGETS)),)
long strings, maybe let's s@/hw/perfromance@/perf@ ?
> ifeq ($(filter net/lib,$(TARGETS)),)
> INSTALL_DEP_TARGETS := net/lib
> endif
> diff --git a/tools/testing/selftests/drivers/net/hw/performance/Makefile b/tools/testing/selftests/drivers/net/hw/performance/Makefile
> new file mode 100644
> index 000000000000..5d53ef548306
> --- /dev/null
> +++ b/tools/testing/selftests/drivers/net/hw/performance/Makefile
> @@ -0,0 +1,10 @@
> +# SPDX-License-Identifier: GPL-2.0
> +
> +TEST_PROGS := iperf.py
here..
> +TEST_INCLUDES := \
> + $(wildcard ../lib/py/*.py ../../lib/py/*.py) \
> + $(wildcard ../../../../net/lib/py/*.py) \
> + #
.. and here -
not the exact correct format for the Makefile, pls look closer
> +include ../../../../lib.mk
> diff --git a/tools/testing/selftests/drivers/net/hw/performance/config b/tools/testing/selftests/drivers/net/hw/performance/config
> new file mode 100644
> index 000000000000..6450f34933e6
> --- /dev/null
> +++ b/tools/testing/selftests/drivers/net/hw/performance/config
> @@ -0,0 +1 @@
> +CONFIG_IPV6=y
> diff --git a/tools/testing/selftests/drivers/net/hw/performance/iperf.py b/tools/testing/selftests/drivers/net/hw/performance/iperf.py
> new file mode 100755
> index 000000000000..415fb67f4100
> --- /dev/null
> +++ b/tools/testing/selftests/drivers/net/hw/performance/iperf.py
> @@ -0,0 +1,27 @@
> +#!/usr/bin/env python3
> +# SPDX-License-Identifier: GPL-2.0
> +
> +"""Measure TCP throughput with iperf3."""
> +
> +from os import path
> +import sys
> +
> +sys.path.insert(0, path.join(path.dirname(path.abspath(__file__)), ".."))
Nasty hacks. Something is not laid out right, mister.
> +from lib.py import Iperf3Runner, NetDrvEpEnv, ksft_exit, ksft_run
> +
> +
> +def tcp(cfg):
> + """Measure TCP throughput."""
> + Iperf3Runner(cfg).measure_bandwidth(report=True)
> +
> +
> +def main():
> + """Run the iperf performance test."""
> + with NetDrvEpEnv(__file__, nsim_test=False) as cfg:
> + ksft_run([tcp], args=(cfg,))
> + ksft_exit()
> +
> +
> +if __name__ == "__main__":
> + main()
> diff --git a/tools/testing/selftests/drivers/net/hw/performance/settings b/tools/testing/selftests/drivers/net/hw/performance/settings
> new file mode 100644
> index 000000000000..e7b9417537fb
> --- /dev/null
> +++ b/tools/testing/selftests/drivers/net/hw/performance/settings
> @@ -0,0 +1 @@
> +timeout=0
> diff --git a/tools/testing/selftests/drivers/net/lib/py/load.py b/tools/testing/selftests/drivers/net/lib/py/load.py
> index e24660e5c27f..a7da025b4e21 100644
> --- a/tools/testing/selftests/drivers/net/lib/py/load.py
> +++ b/tools/testing/selftests/drivers/net/lib/py/load.py
> @@ -4,7 +4,8 @@ import re
> import time
> import json
>
> -from lib.py import ksft_pr, cmd, ip, rand_port, wait_port_listen
> +from lib.py import cmd, ip, ksft_metric, ksft_metric_policy, ksft_pr, \
> + rand_port, wait_port_listen
>
>
> class Iperf3Runner:
> @@ -53,12 +54,20 @@ from lib.py import ksft_pr, cmd, ip, rand_port, wait_port_listen
> kwargs["timeout"] = duration + 5
> return cmd(cmdline, **kwargs)
>
> - def measure_bandwidth(self, reverse=False):
> + def measure_bandwidth(self, reverse=False, report=False):
> """
> Runs an iperf3 measurement and returns the average bandwidth (Gbps).
> Discards the first and last few reporting intervals and uses only the
> middle part of the run where throughput is typically stable.
> """
> + if report:
> + ksft_metric_policy(
> + "throughput", summarize="p50",
> + regression={"compare": "p50", "better": "higher",
> + "relative_tolerance": 0.05},
> + display_label="Gbps")
> + ksft_metric_policy("tcp.retransmits", summarize="max")
> +
> self.start_server()
> result = self.start_client(duration=10, reverse=reverse)
>
> @@ -77,6 +86,17 @@ from lib.py import ksft_pr, cmd, ip, rand_port, wait_port_listen
> stable = samples[3:-3]
>
> avg = sum(stable) / len(stable)
> + retransmits = out.get("end", {}).get("sum_sent", {}).get(
> + "retransmits")
> + if report:
> + if not isinstance(retransmits, int) or \
> + isinstance(retransmits, bool):
> + raise ValueError("iperf3 did not report TCP retransmits")
> + receiver = "remote" if reverse else "local"
> + sender = "local" if reverse else "remote"
> + ksft_metric("throughput", avg, kind="scalar", host=receiver)
> + ksft_metric("tcp.retransmits", retransmits, kind="scalar",
> + host=sender)
I'd be tempted to find a way to pass a callback to the class
for result reporting rather than teach the class how to report.
But just a thought, not sure which ends up being cleaner.
^ permalink raw reply [flat|nested] 10+ messages in thread
* [RFC net-next v2 4/5] selftests: drv-net: add a kperf runner
2026-10-05 20:11 [RFC net-next v2 0/5] selftests: net: add performance metric reporting Stanislav Fomichev
` (2 preceding siblings ...)
2026-10-05 20:11 ` [RFC net-next v2 3/5] selftests: drv-net: add an iperf performance test Stanislav Fomichev
@ 2026-10-05 20:11 ` Stanislav Fomichev
2026-10-16 17:05 ` Jakub Kicinski
2026-10-05 20:11 ` [RFC net-next v2 5/5] selftests: drv-net: measure devmem performance with kperf Stanislav Fomichev
4 siblings, 1 reply; 10+ messages in thread
From: Stanislav Fomichev @ 2026-10-05 20:11 UTC (permalink / raw)
To: netdev; +Cc: davem, edumazet, kuba, pabeni
Add a helper to manage kperf servers, run the client, and optionally report
aggregate throughput and retransmits. Use a random control port and a
unique PID file, and report server cleanup failures.
Signed-off-by: Stanislav Fomichev <sdf@fomichev.me>
---
.../testing/selftests/drivers/net/README.rst | 8 +
.../drivers/net/hw/lib/py/__init__.py | 6 +-
.../selftests/drivers/net/lib/py/__init__.py | 4 +-
.../selftests/drivers/net/lib/py/kperf.py | 161 ++++++++++++++++++
4 files changed, 176 insertions(+), 3 deletions(-)
create mode 100644 tools/testing/selftests/drivers/net/lib/py/kperf.py
diff --git a/tools/testing/selftests/drivers/net/README.rst b/tools/testing/selftests/drivers/net/README.rst
index 1b7264d40b79..4355d5113ec8 100644
--- a/tools/testing/selftests/drivers/net/README.rst
+++ b/tools/testing/selftests/drivers/net/README.rst
@@ -141,6 +141,14 @@ Arguments used to construct the communication channel.
for netns - name of the "remote" namespace
for ssh - name/address of the remote host
+kperf
+~~~~~
+
+Performance tests which use kperf expect ``kperf-client`` in ``PATH`` on the
+local host and ``kperf-server`` in ``PATH`` on both endpoints. Kperf is an
+optional dependency; tests which require it are skipped when it is not
+installed.
+
Example
=======
diff --git a/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py b/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py
index cd771c00c074..c3d84d0f82b9 100644
--- a/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py
+++ b/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py
@@ -33,7 +33,8 @@ KSFT_DIR = (Path(__file__).parent / "../../../../..").resolve()
from net.lib.py import ksft_eq, ksft_ge, ksft_in, ksft_is, ksft_lt, \
ksft_ne, ksft_not_in, ksft_raises, ksft_true, ksft_gt, ksft_not_none
from drivers.net.lib.py import set_ethtool_feat
- from drivers.net.lib.py import GenerateTraffic, Remote, Iperf3Runner
+ from drivers.net.lib.py import GenerateTraffic, Remote, Iperf3Runner, \
+ KperfResult, KperfRunner
from drivers.net.lib.py import NetDrvEnv, NetDrvEpEnv, NetDrvContEnv
__all__ = ["NetNS", "NetNSEnter", "NetdevSimDev", "UserNetNS",
@@ -53,7 +54,8 @@ KSFT_DIR = (Path(__file__).parent / "../../../../..").resolve()
"ksft_ne", "ksft_not_in", "ksft_raises", "ksft_true", "ksft_gt",
"ksft_not_none", "ksft_not_none",
"NetDrvEnv", "NetDrvEpEnv", "NetDrvContEnv", "GenerateTraffic",
- "Remote", "Iperf3Runner", "set_ethtool_feat"]
+ "Remote", "Iperf3Runner", "KperfResult", "KperfRunner",
+ "set_ethtool_feat"]
except ModuleNotFoundError as e:
print("Failed importing `net` library from kernel sources")
print(str(e))
diff --git a/tools/testing/selftests/drivers/net/lib/py/__init__.py b/tools/testing/selftests/drivers/net/lib/py/__init__.py
index 58c2877e7928..e428d0ac1a09 100644
--- a/tools/testing/selftests/drivers/net/lib/py/__init__.py
+++ b/tools/testing/selftests/drivers/net/lib/py/__init__.py
@@ -52,11 +52,13 @@ KSFT_DIR = (Path(__file__).parent / "../../../..").resolve()
from .env import NetDrvEnv, NetDrvEpEnv, NetDrvContEnv
from .feat import set_ethtool_feat
+ from .kperf import KperfResult, KperfRunner
from .load import GenerateTraffic, Iperf3Runner
from .remote import Remote
__all__ += ["NetDrvEnv", "NetDrvEpEnv", "NetDrvContEnv", "GenerateTraffic",
- "Remote", "Iperf3Runner", "set_ethtool_feat"]
+ "Remote", "Iperf3Runner", "KperfResult", "KperfRunner",
+ "set_ethtool_feat"]
except ModuleNotFoundError as e:
print("Failed importing `net` library from kernel sources")
print(str(e))
diff --git a/tools/testing/selftests/drivers/net/lib/py/kperf.py b/tools/testing/selftests/drivers/net/lib/py/kperf.py
new file mode 100644
index 000000000000..f0149a50fdc9
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/lib/py/kperf.py
@@ -0,0 +1,161 @@
+# SPDX-License-Identifier: GPL-2.0
+
+"""Run and parse kperf network performance measurements."""
+
+import re
+import shlex
+import subprocess
+import uuid
+from dataclasses import dataclass
+
+from lib.py import cmd, ksft_metric, ksft_metric_policy, ksft_pr, \
+ rand_port, wait_port_listen
+
+
+_SECTION_RE = re.compile(r"==\s+(Source|Target)\b")
+_RATE_RE = re.compile(r"\b(Tx|Rx)\s*([0-9]+(?:\.[0-9]+)?)\s+Gbps\b")
+_TCP_HEADER = "TCP retrans reord rtt rttvar d_ce snd_wnd cwnd"
+_TCP_ROW_RE = re.compile(r"^\s*(?:[^:\s]+:\s*)?(\d+)(?:\s+\d+){6}\s*$")
+
+
+@dataclass
+class KperfResult:
+ """Aggregate throughput (Gbps) and retransmits of both endpoints."""
+
+ source_throughput: float
+ target_throughput: float
+ source_retransmits: int
+ target_retransmits: int
+
+
+class KperfRunner:
+ """Manage kperf servers and run measurements between test endpoints."""
+
+ def __init__(self, env):
+ self.env = env
+ self._pid_file = f"/tmp/ksft-kperf-{uuid.uuid4().hex[:8]}.pid"
+ self._port = rand_port()
+ self._started = []
+ env.require_cmd("kperf-client", local=True)
+ env.require_cmd("kperf-server", local=True, remote=True)
+
+ def _server(self, remote, kill=False):
+ host = self.env.remote if remote else None
+ address = self.env.remote_addr if remote else self.env.addr
+ args = ["kperf-server", "--pid-file", self._pid_file]
+ if kill:
+ args.append("--kill")
+ else:
+ args += ["--addr", address, "--port", str(self._port)]
+ return cmd(shlex.join(args) if remote else args, host=host,
+ fail=not kill)
+
+ def __enter__(self):
+ try:
+ for remote in (False, True):
+ self._server(remote)
+ self._started.append(remote)
+ wait_port_listen(self._port)
+ wait_port_listen(self._port, host=self.env.remote)
+ except Exception:
+ self.close()
+ raise
+ return self
+
+ def close(self):
+ """Stop the started kperf servers."""
+ failures = []
+ while self._started:
+ command = self._server(self._started.pop(), kill=True)
+ if command.ret:
+ failures.append(command)
+ if failures:
+ details = "\n".join(str(command) for command in failures)
+ raise RuntimeError(f"failed to stop kperf server(s)\n{details}")
+
+ def __exit__(self, _exc_type, _exc_value, _exc_tb):
+ self.close()
+
+ def run(self, client_args=(), report=False):
+ """Run kperf and optionally report throughput and retransmits."""
+ argv = ["kperf-client", "--src", self.env.remote_addr,
+ "--dst", self.env.addr, "--src-svc", str(self._port),
+ "--dst-svc", str(self._port)]
+ argv += [str(arg) for arg in client_args]
+ command = cmd(argv, background=True)
+ try:
+ command.process(terminate=False, fail=False, timeout=300)
+ except subprocess.TimeoutExpired as error:
+ command.proc.kill()
+ command.process(terminate=False, fail=False)
+ raise RuntimeError(f"kperf client timed out\n{command!r}") \
+ from error
+ if command.ret:
+ raise RuntimeError(
+ f"kperf client exited with status {command.ret}\n{command!r}")
+
+ try:
+ result = self.parse(command.stdout + "\n" + command.stderr)
+ except ValueError:
+ ksft_pr(command)
+ raise
+
+ if report:
+ ksft_metric_policy(
+ "throughput", summarize="p50",
+ regression={"compare": "p50", "better": "higher",
+ "relative_tolerance": 0.05},
+ display_label="Gbps")
+ ksft_metric_policy("tcp.retransmits", summarize="max")
+ for host, throughput, retransmits in (
+ ("remote", result.source_throughput,
+ result.source_retransmits),
+ ("local", result.target_throughput,
+ result.target_retransmits)):
+ ksft_metric("throughput", throughput, kind="scalar",
+ host=host)
+ ksft_metric("tcp.retransmits", retransmits, kind="scalar",
+ host=host)
+ return result
+
+ @staticmethod
+ def parse(output):
+ """Parse human-readable kperf client output."""
+ throughput = {"Source": [], "Target": []}
+ retransmits = {"Source": [], "Target": []}
+ section = None
+ in_tcp = False
+
+ for line in output.splitlines():
+ match = _SECTION_RE.search(line)
+ if match:
+ section = match.group(1)
+ in_tcp = False
+ continue
+ if section is None:
+ continue
+
+ match = _RATE_RE.search(line)
+ if match:
+ direction, value = match.groups()
+ if (section, direction) in (("Source", "Tx"),
+ ("Target", "Rx")):
+ throughput[section].append(float(value))
+ continue
+
+ if _TCP_HEADER in line:
+ in_tcp = True
+ continue
+ if in_tcp:
+ match = _TCP_ROW_RE.match(line)
+ if match:
+ retransmits[section].append(int(match.group(1)))
+
+ values = (*throughput.values(), *retransmits.values())
+ if not all(values):
+ raise ValueError("incomplete kperf result")
+
+ return KperfResult(sum(throughput["Source"]),
+ sum(throughput["Target"]),
+ sum(retransmits["Source"]),
+ sum(retransmits["Target"]))
--
2.53.0-Meta
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [RFC net-next v2 4/5] selftests: drv-net: add a kperf runner
2026-10-05 20:11 ` [RFC net-next v2 4/5] selftests: drv-net: add a kperf runner Stanislav Fomichev
@ 2026-10-16 17:05 ` Jakub Kicinski
0 siblings, 0 replies; 10+ messages in thread
From: Jakub Kicinski @ 2026-10-16 17:05 UTC (permalink / raw)
To: Stanislav Fomichev; +Cc: netdev, davem, edumazet, pabeni
On Mon, 5 Oct 2026 13:11:13 -0700 Stanislav Fomichev wrote:
> Add a helper to manage kperf servers, run the client, and optionally report
> aggregate throughput and retransmits. Use a random control port and a
> unique PID file, and report server cleanup failures.
>
> Signed-off-by: Stanislav Fomichev <sdf@fomichev.me>
> ---
> .../testing/selftests/drivers/net/README.rst | 8 +
> .../drivers/net/hw/lib/py/__init__.py | 6 +-
> .../selftests/drivers/net/lib/py/__init__.py | 4 +-
> .../selftests/drivers/net/lib/py/kperf.py | 161 ++++++++++++++++++
> 4 files changed, 176 insertions(+), 3 deletions(-)
> create mode 100644 tools/testing/selftests/drivers/net/lib/py/kperf.py
>
> diff --git a/tools/testing/selftests/drivers/net/README.rst b/tools/testing/selftests/drivers/net/README.rst
> index 1b7264d40b79..4355d5113ec8 100644
> --- a/tools/testing/selftests/drivers/net/README.rst
> +++ b/tools/testing/selftests/drivers/net/README.rst
> @@ -141,6 +141,14 @@ Arguments used to construct the communication channel.
> for netns - name of the "remote" namespace
> for ssh - name/address of the remote host
>
> +kperf
> +~~~~~
> +
> +Performance tests which use kperf expect ``kperf-client`` in ``PATH`` on the
> +local host and ``kperf-server`` in ``PATH`` on both endpoints. Kperf is an
> +optional dependency; tests which require it are skipped when it is not
> +installed.
I don't think this belongs in the README.
README needs to be high signal, otherwise nobody will read.
People running perf tests will quickly figure out kperf by asking LLM
^ permalink raw reply [flat|nested] 10+ messages in thread
* [RFC net-next v2 5/5] selftests: drv-net: measure devmem performance with kperf
2026-10-05 20:11 [RFC net-next v2 0/5] selftests: net: add performance metric reporting Stanislav Fomichev
` (3 preceding siblings ...)
2026-10-05 20:11 ` [RFC net-next v2 4/5] selftests: drv-net: add a kperf runner Stanislav Fomichev
@ 2026-10-05 20:11 ` Stanislav Fomichev
4 siblings, 0 replies; 10+ messages in thread
From: Stanislav Fomichev @ 2026-10-05 20:11 UTC (permalink / raw)
To: netdev; +Cc: davem, edumazet, kuba, pabeni
Add a separate udmabuf-backed devmem performance test using KperfRunner.
Keep correctness cases in drivers/net/hw. Skip the test when kperf or
/dev/udmabuf is missing, and fail it when the kperf run fails.
Example metric output:
# # ktap-metric-policy-json: {"display_label":"Gbps","name":"throughput","regression":{"better":"higher","compare":"p50","relative_tolerance":0.05},"summarize":"p50"}
# # ktap-metric-policy-json: {"name":"tcp.retransmits","summarize":"max"}
# # ktap-metric-json: {"host":"remote","name":"throughput","time":29.786267,"type":"scalar","value":389.29900000000004}
# # ktap-metric-json: {"host":"remote","name":"tcp.retransmits","time":29.786286,"type":"scalar","value":698}
# # ktap-metric-json: {"host":"local","name":"throughput","time":29.786289,"type":"scalar","value":388.772}
# # ktap-metric-json: {"host":"local","name":"tcp.retransmits","time":29.78629,"type":"scalar","value":0}
Signed-off-by: Stanislav Fomichev <sdf@fomichev.me>
---
.../drivers/net/hw/performance/Makefile | 5 +-
.../drivers/net/hw/performance/config | 3 +
.../drivers/net/hw/performance/devmem.py | 99 +++++++++++++++++++
3 files changed, 106 insertions(+), 1 deletion(-)
create mode 100755 tools/testing/selftests/drivers/net/hw/performance/devmem.py
diff --git a/tools/testing/selftests/drivers/net/hw/performance/Makefile b/tools/testing/selftests/drivers/net/hw/performance/Makefile
index 5d53ef548306..27afdace92e6 100644
--- a/tools/testing/selftests/drivers/net/hw/performance/Makefile
+++ b/tools/testing/selftests/drivers/net/hw/performance/Makefile
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: GPL-2.0
-TEST_PROGS := iperf.py
+TEST_PROGS := \
+ devmem.py \
+ iperf.py \
+ #
TEST_INCLUDES := \
$(wildcard ../lib/py/*.py ../../lib/py/*.py) \
diff --git a/tools/testing/selftests/drivers/net/hw/performance/config b/tools/testing/selftests/drivers/net/hw/performance/config
index 6450f34933e6..111e9b58610d 100644
--- a/tools/testing/selftests/drivers/net/hw/performance/config
+++ b/tools/testing/selftests/drivers/net/hw/performance/config
@@ -1 +1,4 @@
CONFIG_IPV6=y
+CONFIG_NET_DEVMEM=y
+CONFIG_SYNC_FILE=y
+CONFIG_UDMABUF=y
diff --git a/tools/testing/selftests/drivers/net/hw/performance/devmem.py b/tools/testing/selftests/drivers/net/hw/performance/devmem.py
new file mode 100755
index 000000000000..033485f926e4
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/performance/devmem.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+
+"""Measure udmabuf-backed devmem TCP throughput with kperf."""
+
+from os import path
+import sys
+
+sys.path.insert(0, path.join(path.dirname(path.abspath(__file__)), ".."))
+
+from lib.py import cmd, defer, ethtool, EthtoolFamily, KperfRunner
+from lib.py import ksft_disruptive, ksft_exit, ksft_run, KsftSkipEx
+from lib.py import NetDrvEpEnv
+
+
+UDMABUF_LIMIT = "/sys/module/udmabuf/parameters/size_limit_mb"
+UDMABUF_SIZE_MB = 2048
+
+
+def _set_udmabuf_limit(host) -> None:
+ current = cmd(f"cat {UDMABUF_LIMIT}", host=host).stdout.strip()
+ if int(current) >= UDMABUF_SIZE_MB:
+ return
+
+ cmd(f"echo {UDMABUF_SIZE_MB} > {UDMABUF_LIMIT}", host=host, shell=True)
+ defer(cmd, f"echo {current} > {UDMABUF_LIMIT}", host=host, shell=True)
+
+
+def _enable_hw_gro(cfg) -> None:
+ features = ethtool(f"-k {cfg.ifname}", json=True)[0]
+ hw_gro = features["rx-gro-hw"]
+ if not hw_gro["active"]:
+ if hw_gro["fixed"]:
+ raise KsftSkipEx("Test requires rx-gro-hw support")
+ ethtool(f"-K {cfg.ifname} rx-gro-hw on")
+ defer(ethtool, f"-K {cfg.ifname} rx-gro-hw off")
+
+
+@ksft_disruptive
+def kperf(cfg) -> None:
+ """Measure udmabuf-backed devmem performance with kperf."""
+ runner = KperfRunner(cfg)
+
+ for name, host in (("local", None), ("remote", cfg.remote)):
+ if cmd("test -c /dev/udmabuf", host=host, fail=False).ret:
+ raise KsftSkipEx(f"Test requires /dev/udmabuf on the {name} host")
+ _set_udmabuf_limit(host)
+
+ # kperf reprograms the RSS indirection table for its queues.
+ defer(ethtool, f"-X {cfg.ifname} default")
+
+ _enable_hw_gro(cfg)
+
+ ethnl = EthtoolFamily()
+ rings = ethnl.rings_get({"header": {"dev-index": cfg.ifindex}})
+ restore = {
+ "header": {"dev-index": cfg.ifindex},
+ "tcp-data-split": rings.get("tcp-data-split", "unknown"),
+ }
+ config = {
+ "header": {"dev-index": cfg.ifindex},
+ "tcp-data-split": "enabled",
+ }
+ # Memory providers require a zero threshold where the driver has one.
+ if "hds-thresh" in rings:
+ restore["hds-thresh"] = rings["hds-thresh"]
+ config["hds-thresh"] = 0
+ defer(ethnl.rings_set, restore)
+ ethnl.rings_set(config)
+
+ devmem_args = [
+ "--time", "15",
+ "--num-connections", "4",
+ "--read-size", str(64 << 20),
+ "--write-size", str(64 << 20),
+ "--pin-off", "4",
+ "--num-rx-queues", "4",
+ "--dmabuf-rx-size-mb", str(UDMABUF_SIZE_MB),
+ "--dmabuf-tx-size-mb", str(UDMABUF_SIZE_MB),
+ "--validate", "no",
+ "--devmem-rx",
+ "--devmem-tx",
+ "--devmem-rx-memory", "host",
+ "--devmem-tx-memory", "host",
+ ]
+
+ with runner:
+ runner.run(client_args=devmem_args, report=True)
+
+
+def main() -> None:
+ """Run the devmem performance test."""
+ with NetDrvEpEnv(__file__, nsim_test=False) as cfg:
+ ksft_run([kperf], args=(cfg,))
+ ksft_exit()
+
+
+if __name__ == "__main__":
+ main()
--
2.53.0-Meta
^ permalink raw reply related [flat|nested] 10+ messages in thread