[PATCH v1] dts: fix TRex packet stream handling for line rate testing

[PATCH v1] dts: fix TRex packet stream handling for line rate testing

[Dillon Ladd]

Packets were not reaching line rate on certain hardware configurations.
Investigation revealed two root causes in the TRex traffic generator
implementation.

The first issue was that a single unidirectional packet stream could not
adequately represent bidirectional traffic patterns. The second issue was
that TRex ports were not configured with promiscuous mode enabled. When
network devices forward traffic without rewriting L2 addresses, the return
traffic arrives at the traffic generator addressed to the device's ingress
MAC rather than the traffic generator's port MAC. Without promiscuous mode,
the traffic generator discards these packets and reports incomplete
statistics.

This patch addresses both issues by refactoring packet stream creation to
generate two streams per direction, each operating at 50% port capacity for
balanced traffic distribution. Promiscuous mode is now explicitly enabled
on all ports during TRex client setup. Additionally, error handling in the
traffic start procedure has been improved to better distinguish failure
modes and provide more informative diagnostics.

Bugzilla ID: 1977
Fixes: d77d7f04f24c ("dts: add single-core performance test suite")

Signed-off-by: Andrew Bailey <abailey@iol.unh.edu>
Signed-off-by: Dillon Ladd <dladd@iol.unh.edu>
---
 .mailmap                                      |  1 +
 dts/api/packet.py                             |  1 +
 .../testbed_model/traffic_generator/trex.py   | 97 +++++++++++++------
 3 files changed, 69 insertions(+), 30 deletions(-)

diff --git a/.mailmap b/.mailmap
index fcb3d1bb3f..58107ffe3a 100644
--- a/.mailmap
+++ b/.mailmap
@@ -402,6 +402,7 @@ Dheemanth Mallikarjun <dheemanthm@vmware.com>
 Dhruv Tripathi <dhruv.tripathi@arm.com>
 Diana Wang <na.wang@corigine.com>
 Didier Pallard <didier.pallard@6wind.com>
+Dillon Ladd <dladd@iol.unh.edu>
 Dilshod Urazov <dilshod.urazov@oktetlabs.ru>
 Dima Ruinskiy <dima.ruinskiy@intel.com>
 Dimon Zhao <dimon.zhao@nebula-matrix.com>
diff --git a/dts/api/packet.py b/dts/api/packet.py
index 3dda18e781..9a8c081621 100644
--- a/dts/api/packet.py
+++ b/dts/api/packet.py
@@ -348,4 +348,5 @@ def assess_performance_by_packet(
     )
     tg: PerformanceTrafficGenerator = cast(PerformanceTrafficGenerator, get_ctx().perf_tg)
     # TODO: implement @requires for types of traffic generator
+    packet = adjust_addresses([packet])[0]
     return tg.calculate_traffic_and_stats(packet, duration, send_mpps)
diff --git a/dts/framework/testbed_model/traffic_generator/trex.py b/dts/framework/testbed_model/traffic_generator/trex.py
index d53791b0a6..eceb79b1ee 100644
--- a/dts/framework/testbed_model/traffic_generator/trex.py
+++ b/dts/framework/testbed_model/traffic_generator/trex.py
@@ -9,6 +9,7 @@
 from enum import auto
 from typing import ClassVar
 
+from scapy.layers.inet import IP
 from scapy.packet import Packet
 
 from framework.config.node import OS, NodeConfiguration
@@ -66,7 +67,6 @@ class TrexTrafficGenerator(PerformanceTrafficGenerator):
 
     Attributes:
         stl_client_name: The name of the stateless client used in the stateless API.
-        packet_stream_name: The name of the stateless packet stream used in the stateless API.
     """
 
     _os_session: OSSession
@@ -78,7 +78,6 @@ class TrexTrafficGenerator(PerformanceTrafficGenerator):
     _python_indentation: ClassVar[str] = " " * 4
 
     stl_client_name: ClassVar[str] = "client"
-    packet_stream_name: ClassVar[str] = "stream"
 
     _streaming_mode: TrexStatelessTXModes = TrexStatelessTXModes.STLTXCont
 
@@ -86,6 +85,9 @@ class TrexTrafficGenerator(PerformanceTrafficGenerator):
 
     _trex_app: BlockingApp
 
+    _topology: Topology
+    _stream_names_by_port: dict[int, list[str]]
+
     def __init__(self, tg_node: Node, config: TrexTrafficGeneratorConfig) -> None:
         """Initialize the TRex server.
 
@@ -108,6 +110,7 @@ def __init__(self, tg_node: Node, config: TrexTrafficGeneratorConfig) -> None:
     def setup(self, topology: Topology):
         """Initialize and start a TRex server process."""
         super().setup(topology)
+        self._topology = topology
 
         self._shell = PythonShell(self._tg_node, "TRex-client", privileged=True)
 
@@ -196,30 +199,54 @@ def _setup_trex_client(self) -> None:
         procedure = [
             f"{self.stl_client_name}.connect()",
             f"{self.stl_client_name}.reset(ports = [0, 1])",
+            f"{self.stl_client_name}.set_port_attr(ports=[0, 1], promiscuous=True)",
             f"{self.stl_client_name}.clear_stats()",
-            f"{self.stl_client_name}.add_streams({self.packet_stream_name}, ports=[0, 1])",
         ]
 
+        for port_id, stream_names in self._stream_names_by_port.items():
+            procedure.append(
+                f"{self.stl_client_name}.add_streams("
+                f"[{', '.join(stream_names)}], ports=[{port_id}])"
+            )
+
         for command in procedure:
             self._shell.send_command(command)
 
     def _create_packet_stream(self, packet: Packet) -> None:
-        """Create TRex packet stream with the given packet.
+        """Create TRex packet streams, two per direction.
 
         Args:
             packet: The packet being used for the performance test.
         """
-        # Create the tx packet on the TG shell
-        self._shell.send_command(f"packet={packet.command()}")
-
-        packet_stream = [
-            f"{self.packet_stream_name} = trex.stl.trex_stl_streams.STLStream(",
-            f"name='Test_{len(packet)}_bytes',",
-            "packet=trex.stl.trex_stl_packet_builder_scapy.STLPktBuilder(pkt=packet),",
-            f"mode=trex.stl.trex_stl_streams.{self._streaming_mode}(percentage=100),",
-            ")",
-        ]
-        self._shell.send_command("\n".join(packet_stream))
+        forward: Packet = packet.copy()
+        reverse: Packet = packet.copy()
+        reverse.src = "52:00:00:00:00:00"
+        reverse.dst = self._topology.sut_port_egress.mac_address
+
+        self._stream_names_by_port = {}
+
+        for port_id, direction_packet in ((0, forward), (1, reverse)):
+            stream_names: list[str] = []
+            ip_prefix: str = direction_packet[IP].dst.rsplit(".", 1)[0]
+
+            for index in range(2):
+                packet_var = f"packet_p{port_id}_{index}"
+                stream_var = f"stream_p{port_id}_{index}"
+                direction_packet[IP].dst = f"{ip_prefix}.{index}"
+                self._shell.send_command(f"{packet_var}={direction_packet.command()}")
+
+                # percentage=50 per stream: two streams share the port.
+                stream_definition = [
+                    f"{stream_var} = trex.stl.trex_stl_streams.STLStream(",
+                    f"name='Test_{len(packet)}_bytes_p{port_id}_{index}',",
+                    "packet=trex.stl.trex_stl_packet_builder_scapy.STLPktBuilder("
+                    f"pkt={packet_var}),",
+                    f"mode=trex.stl.trex_stl_streams.{self._streaming_mode}(percentage=50),",
+                    ")",
+                ]
+                self._shell.send_command("\n".join(stream_definition))
+                stream_names.append(stream_var)
+            self._stream_names_by_port[port_id] = stream_names
 
     def _send_traffic_and_get_stats(
         self, duration: float, send_mpps: float | None = None, retry_attempts: int = 5
@@ -236,28 +263,38 @@ def _send_traffic_and_get_stats(
             retry_attempts: The number of times to retry this command on failure.
 
         Raises:
-            SSHTimeoutError: If TRex fails to send traffic in the allotted attempts.
+            SSHTimeoutError: If TRex refuses to start traffic in the allotted attempts.
         """
-        link_down = True
+        started = False
         attempt = 0
 
-        while link_down and attempt < retry_attempts:
-            if send_mpps:
-                result = self._shell.send_command(f"""{self.stl_client_name}.start(ports=[0, 1],
-                    mult = '{send_mpps}mpps',
-                    duration = {duration})""")
-            else:
-                result = self._shell.send_command(f"""{self.stl_client_name}.start(ports=[0, 1],
-                    mult = '100%',
-                    duration = {duration})""")
-            link_down = "link is DOWN" in result
-            if link_down:
+        mult = f"'{send_mpps}mpps'" if send_mpps else "'100%'"
+        while not started and attempt < retry_attempts:
+            # Capture the RC and verify that a refused start can never silently produce a
+            # 0.0 Mpps measurement.
+            self._shell.send_command("_start_rc = None")
+            result = self._shell.send_command(
+                f"_start_rc = {self.stl_client_name}.start(ports=[0, 1], "
+                f"mult={mult}, duration={duration})"
+            )
+            ok = self._shell.send_command(
+                "bool(_start_rc is not None"
+                " and (not hasattr(_start_rc, 'good') or _start_rc.good()))"
+            )
+            started = "True" in ok
+            if not started:
+                error_text = result + self._shell.send_command(
+                    "str(getattr(_start_rc, 'err', lambda: '')()) if _start_rc is not None else ''"
+                )
                 self._logger.info(
-                    f"Generate traffic command failed (attempt {attempt + 1} of {retry_attempts})"
+                    f"Traffic start refused (attempt {attempt + 1} of {retry_attempts}): "
+                    f"{error_text.strip()}"
                 )
+                if "link is DOWN" not in error_text:
+                    raise SSHTimeoutError(f"TRex refused to start traffic: {error_text.strip()}")
                 time.sleep(0.25)
             attempt += 1
-        if link_down:
+        if not started:
             raise SSHTimeoutError("Link failed to come up, Traffic could not be generated.")
 
         time.sleep(duration)
-- 
2.53.0

Re: [PATCH v1] dts: fix TRex packet stream handling for line rate testing

[Luca Vizzarro]

Hi Dillon and Andrew,

Thank you for your patch and so sorry for the wait! This looks good to 
me, I have no objections against it.

Reviewed-by: Luca Vizzarro <luca.vizzarro@arm.com>

Luca