[Autotest PATCH v2 2/4] virt-test: Refactor netperf test and add analysis module

[Amos Kong <akong@redhat.com>]

Always use a VM as netperf server, we can use
another VM/localhost/external host as the netperf
clients.
We setup env and launch test by executing remote
ssh commands, you need to configure the IP of
local/external host in configure file, VMs' IP
can be got automatically.
Generate a file with 'standard' format at the end of test,
then we can analyze them by general module.

Changes from v1:
- record packet bytes
- enable arp_ignore
- get packet info from ifconfig
- shape functions
- don't change ssh config

Signed-off-by: Amos Kong <akong@redhat.com>
---
 client/virt/subtests.cfg.sample |   36 +++--
 client/virt/tests/netperf.py    |  295 +++++++++++++++++++++++++++------------
 2 files changed, 233 insertions(+), 98 deletions(-)

diff --git a/client/virt/subtests.cfg.sample b/client/virt/subtests.cfg.sample
index 5b41f67..fc2621d 100644
--- a/client/virt/subtests.cfg.sample
+++ b/client/virt/subtests.cfg.sample
@@ -992,20 +992,36 @@ variants:
 
     - netperf: install setup image_copy unattended_install.cdrom
         only Linux
+        only virtio_net
         type = netperf
-        nics += ' nic2 nic3 nic4'
+        kill_vm = yes
+        image_snapshot = yes
+        nics += ' nic2'
+        # nic1 is for control, nic2 is for data connection
+        # bridge_nic1 = virbr0
+        pci_model_nic1 = virtio_net
+        # bridge_nic2 = switch
+        pci_model_nic2 = e1000
         nic_mode = tap
         netperf_files = netperf-2.4.5.tar.bz2 wait_before_data.patch
-        packet_size = 1500
-        setup_cmd = "cd %s && tar xvfj netperf-2.4.5.tar.bz2 && cd netperf-2.4.5 && patch -p0 < ../wait_before_data.patch && ./configure && make"
-        netserver_cmd =  %s/netperf-2.4.5/src/netserver
+        setup_cmd = "cd /tmp && rm -rf netperf-2.4.5 && tar xvfj netperf-2.4.5.tar.bz2 && cd netperf-2.4.5 && patch -p0 < ../wait_before_data.patch && ./configure && make"
+        # configure netperf test parameters
+        # l = 60
+        # protocols = "TCP_STREAM TCP_MAERTS TCP_RR"
+        # sessions = "1 2 4"
+        # sessions_rr = "50 100 250 500"
+        # sizes = "64 256 512 1024"
+        # sizes_rr = "64 256 512 1024"
         variants:
-            - stream:
-                netperf_cmd = %s/netperf-2.4.5/src/netperf -t %s -H %s -l 60 -- -m %s
-                protocols = "TCP_STREAM TCP_MAERTS TCP_SENDFILE UDP_STREAM"
-            - rr:
-                netperf_cmd = %s/netperf-2.4.5/src/netperf -t %s -H %s -l 60 -- -r %s
-                protocols = "TCP_RR TCP_CRR UDP_RR"
+            - guest_guest:
+                vms += " vm2"
+                nics = 'nic1'
+            - host_guest:
+                # local host ip address
+                # client = localhost
+            - exhost_guest:
+                # external host ip address
+                # client =
 
     - ntttcp:
         type = ntttcp
diff --git a/client/virt/tests/netperf.py b/client/virt/tests/netperf.py
index fea1e9e..fd037df 100644
--- a/client/virt/tests/netperf.py
+++ b/client/virt/tests/netperf.py
@@ -1,17 +1,17 @@
-import logging, os, signal
+import logging, os, commands, sys, threading, re, glob
 from autotest_lib.client.common_lib import error
 from autotest_lib.client.bin import utils
 from autotest_lib.client.virt import aexpect, virt_utils
+from autotest_lib.client.virt import virt_test_utils
 
 def run_netperf(test, params, env):
     """
     Network stress test with netperf.
 
-    1) Boot up a VM with multiple nics.
-    2) Launch netserver on guest.
-    3) Execute multiple netperf clients on host in parallel
-       with different protocols.
-    4) Output the test result.
+    1) Boot up VM(s), setup SSH authorization between host
+       and guest(s)/external host
+    2) Prepare the test environment in server/client/host
+    3) Execute netperf tests, collect and analyze the results
 
     @param test: KVM test object.
     @param params: Dictionary with the test parameters.
@@ -21,86 +21,205 @@ def run_netperf(test, params, env):
     vm.verify_alive()
     login_timeout = int(params.get("login_timeout", 360))
     session = vm.wait_for_login(timeout=login_timeout)
+    server = vm.get_address()
+    server_ctl = vm.get_address(1)
     session.close()
-    session_serial = vm.wait_for_serial_login(timeout=login_timeout)
-
-    netperf_dir = os.path.join(os.environ['AUTODIR'], "tests/netperf2")
-    setup_cmd = params.get("setup_cmd")
-
-    firewall_flush = "iptables -F"
-    session_serial.cmd_output(firewall_flush)
-    try:
-        utils.run("iptables -F")
-    except Exception:
-        pass
-
-    for i in params.get("netperf_files").split():
-        vm.copy_files_to(os.path.join(netperf_dir, i), "/tmp")
-
-    try:
-        session_serial.cmd(firewall_flush)
-    except aexpect.ShellError:
-        logging.warning("Could not flush firewall rules on guest")
-
-    session_serial.cmd(setup_cmd % "/tmp", timeout=200)
-    session_serial.cmd(params.get("netserver_cmd") % "/tmp")
-
-    if "tcpdump" in env and env["tcpdump"].is_alive():
-        # Stop the background tcpdump process
-        try:
-            logging.debug("Stopping the background tcpdump")
-            env["tcpdump"].close()
-        except Exception:
-            pass
-
-    def netperf(i=0):
-        guest_ip = vm.get_address(i)
-        logging.info("Netperf_%s: netserver %s" % (i, guest_ip))
-        result_file = os.path.join(test.resultsdir, "output_%s_%s"
-                                   % (test.iteration, i ))
-        list_fail = []
-        result = open(result_file, "w")
-        result.write("Netperf test results\n")
-
-        for p in params.get("protocols").split():
-            packet_size = params.get("packet_size", "1500")
-            for size in packet_size.split():
-                cmd = params.get("netperf_cmd") % (netperf_dir, p,
-                                                   guest_ip, size)
-                logging.info("Netperf_%s: protocol %s" % (i, p))
-                try:
-                    netperf_output = utils.system_output(cmd,
-                                                         retain_output=True)
-                    result.write("%s\n" % netperf_output)
-                except Exception:
-                    logging.error("Test of protocol %s failed", p)
-                    list_fail.append(p)
-
-        result.close()
-        if list_fail:
-            raise error.TestFail("Some netperf tests failed: %s" %
-                                 ", ".join(list_fail))
-
-    try:
-        logging.info("Setup and run netperf clients on host")
-        utils.run(setup_cmd % netperf_dir)
-
-        bg = []
-        nic_num = len(params.get("nics").split())
-        for i in range(nic_num):
-            bg.append(virt_utils.Thread(netperf, (i,)))
-            bg[i].start()
-
-        completed = False
-        while not completed:
-            completed = True
-            for b in bg:
-                if b.isAlive():
-                    completed = False
-    finally:
-        try:
-            for b in bg:
-                if b:
-                    b.join()
-        finally:
-            session_serial.cmd_output("killall netserver")
+
+    if "vm2" in params["vms"]:
+        vm2 = env.get_vm("vm2")
+        vm2.verify_alive()
+        session2 = vm2.wait_for_login(timeout=login_timeout)
+        client = vm2.get_address()
+        session2.close()
+
+    if params.get("client"):
+        client = params["client"]
+    if params.get("host"):
+        host = params["host"]
+    else:
+        cmd = "ifconfig %s|awk 'NR==2 {print $2}'|awk -F: '{print $2}'"
+        host = commands.getoutput(cmd % params["bridge"])
+
+    shell_port = params["shell_port"]
+    password = params["password"]
+    username = params["username"]
+
+    def env_setup(ip):
+        logging.debug("Setup env for %s" % ip)
+        virt_utils.scp_to_remote(ip, shell_port, username, password,
+                      "~/.ssh/id_dsa.pub", "~/.ssh/authorized_keys")
+        ssh_cmd(ip, "service iptables stop")
+        ssh_cmd(ip, "echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore")
+
+        netperf_dir = os.path.join(os.environ['AUTODIR'], "tests/netperf2")
+        for i in params.get("netperf_files").split():
+            virt_utils.scp_to_remote(ip, shell_port, username, password,
+                                     "%s/%s" % (netperf_dir, i), "/tmp/")
+        ssh_cmd(ip, params.get("setup_cmd"))
+
+    logging.info("Prepare env of server/client/host")
+    if not os.path.exists(os.path.expandvars("$HOME/.ssh/id_dsa.pub")):
+        commands.getoutput('yes ""|ssh-keygen -t dsa -q -N ""')
+
+    env_setup(server_ctl)
+    env_setup(client)
+    env_setup(host)
+    logging.info("Start netperf testing ...")
+    start_test(server, server_ctl, host, client, test.resultsdir,
+               l=int(params.get('l')),
+               sessions_rr=params.get('sessions_rr'),
+               sessions=params.get('sessions'),
+               sizes_rr=params.get('sizes_rr'),
+               sizes=params.get('sizes'),
+               protocols=params.get('protocols'))
+
+
+def start_test(server, server_ctl, host, client, resultsdir, l=60,
+               sessions_rr="50 100 250 500", sessions="1 2 4",
+               sizes_rr="64 256 512 1024 2048",
+               sizes="64 256 512 1024 2048 4096",
+               protocols="TCP_STREAM TCP_MAERTS TCP_RR"):
+    """
+    Start to test with different kind of configurations
+
+    @param server: netperf server ip for data connection
+    @param server_ctl: ip to control netperf server
+    @param host: localhost ip
+    @param client: netperf client ip
+    @param resultsdir: directory to restore the results
+    @param l: test duration
+    @param sessions_rr: sessions number list for RR test
+    @param sessions: sessions number list
+    @param sizes_rr: request/response sizes (TCP_RR, UDP_RR)
+    @param sizes: send size (TCP_STREAM, UDP_STREAM)
+    @param protocols: test type
+    """
+
+    def parse_file(file_prefix, raw=""):
+        """ Parse result files and reture throughput total """
+        thu = 0
+        for file in glob.glob("%s.*.nf" % file_prefix):
+            o = commands.getoutput("cat %s |tail -n 1" % file)
+            try:
+                thu += float(o.split()[raw])
+            except:
+                logging.debug(commands.getoutput("cat %s.*" % file_prefix))
+                return -1
+        return thu
+
+    fd = open("%s/netperf-result.RHS" % resultsdir, "w")
+    for protocol in protocols.split():
+        logging.info(protocol)
+        fd.write(protocol+ "\n")
+        row = "%5s|%8s|%10s|%6s|%9s|%10s|%10s|%12s|%12s|%9s|%8s|%8s|%10s|%10s" \
+              "|%11s|%10s" % ("size", "sessions", "throughput", "cpu",
+              "normalize", "#tx-pkts", "#rx-pkts", "#tx-byts", "#rx-byts",
+              "#re-trans", "#tx-intr", "#rx-intr", "#io_exit", "#irq_inj",
+              "#tpkt/#exit", "#rpkt/#irq")
+        logging.info(row)
+        fd.write(row + "\n")
+        if (protocol == "TCP_RR"):
+            sessions_test = sessions_rr.split()
+            sizes_test = sizes_rr.split()
+        else:
+            sessions_test = sessions.split()
+            sizes_test = sizes.split()
+        for i in sizes_test:
+            for j in sessions_test:
+                if (protocol == "TCP_RR"):
+                    ret = launch_client(1, server, server_ctl, host, client, l,
+                    "-t %s -v 0 -P -0 -- -r %s,%s -b %s" % (protocol, i, i, j))
+                    thu = parse_file("/tmp/netperf.%s" % ret['pid'], 0)
+                else:
+                    ret = launch_client(j, server, server_ctl, host, client, l,
+                                     "-C -c -t %s -- -m %s" % (protocol, i))
+                    thu = parse_file("/tmp/netperf.%s" % ret['pid'], 4)
+                cpu = 100 - float(ret['mpstat'].split()[10])
+                normal = thu / cpu
+                pkt_rx_irq = float(ret['rx_pkts']) / float(ret['irq_inj'])
+                pkt_tx_exit = float(ret['tx_pkts']) / float(ret['io_exit'])
+                row = "%5d|%8d|%10.2f|%6.2f|%9.2f|%10d|%10d|%12d|%12d|%9d" \
+                      "|%8d|%8d|%10d|%10d|%11.2f|%10.2f" % (int(i), int(j),
+                      thu, cpu, normal, ret['tx_pkts'], ret['rx_pkts'],
+                      ret['tx_byts'], ret['rx_byts'], ret['re_pkts'],
+                      ret['tx_intr'], ret['rx_intr'], ret['io_exit'],
+                      ret['irq_inj'], pkt_tx_exit, pkt_rx_irq)
+                logging.info(row)
+                fd.write(row + "\n")
+                fd.flush()
+                logging.debug("Remove temporary files")
+                commands.getoutput("rm -f /tmp/netperf.%s.*.nf" % ret['pid'])
+    fd.close()
+
+
+def ssh_cmd(ip, cmd, user="root"):
+    """
+    Execute remote command and return the output
+
+    @param ip: remote machine IP
+    @param cmd: executed command
+    @param user: username
+    """
+    return utils.system_output('ssh -o StrictHostKeyChecking=no -o '
+    'UserKnownHostsFile=/dev/null %s@%s "%s"' % (user, ip, cmd))
+
+
+def launch_client(sessions, server, server_ctl, host, client, l, nf_args):
+    """ Launch netperf clients """
+
+    client_path="/tmp/netperf-2.4.5/src/netperf"
+    server_path="/tmp/netperf-2.4.5/src/netserver"
+    ssh_cmd(server_ctl, "pidof netserver || %s" % server_path)
+    ncpu = ssh_cmd(server_ctl, "cat /proc/cpuinfo |grep processor |wc -l")
+
+    def count_interrupt(name):
+        """
+        @param name: the name of interrupt, such as "virtio0-input"
+        """
+        intr = 0
+        stat = ssh_cmd(server_ctl, "cat /proc/interrupts |grep %s" % name)
+        for cpu in range(int(ncpu)):
+            intr += int(stat.split()[cpu+1])
+        return intr
+
+    def get_state():
+        for i in ssh_cmd(server_ctl, "ifconfig").split("\n\n"):
+            if server in i:
+                nrx = int(re.findall("RX packets:(\d+)", i)[0])
+                ntx = int(re.findall("TX packets:(\d+)", i)[0])
+                nrxb = int(re.findall("RX bytes:(\d+)", i)[0])
+                ntxb = int(re.findall("TX bytes:(\d+)", i)[0])
+        nre = int(ssh_cmd(server_ctl, "grep Tcp /proc/net/snmp|tail -1"
+                 ).split()[12])
+        nrx_intr = count_interrupt("virtio0-input")
+        ntx_intr = count_interrupt("virtio0-output")
+        io_exit = int(ssh_cmd(host, "cat /sys/kernel/debug/kvm/io_exits"))
+        irq_inj = int(ssh_cmd(host, "cat /sys/kernel/debug/kvm/irq_injections"))
+        return [nrx, ntx, nrxb, ntxb, nre, nrx_intr, ntx_intr, io_exit, irq_inj]
+
+    def netperf_thread(i):
+        cmd = "%s -H %s -l %s %s" % (client_path, server, l, nf_args)
+        output = ssh_cmd(client, cmd)
+        f = file("/tmp/netperf.%s.%s.nf" % (pid, i), "w")
+        f.write(output)
+        f.close()
+
+    start_state = get_state()
+    pid = str(os.getpid())
+    threads = []
+    for i in range(int(sessions)):
+        t = threading.Thread(target=netperf_thread, kwargs={"i": i})
+        threads.append(t)
+        t.start()
+    ret = {}
+    ret['pid'] = pid
+    ret['mpstat'] = ssh_cmd(host, "mpstat 1 %d |tail -n 1" % (l - 1))
+    for t in threads:
+        t.join()
+
+    end_state = get_state()
+    items = ['rx_pkts', 'tx_pkts', 'rx_byts', 'tx_byts', 're_pkts',
+             'rx_intr', 'tx_intr', 'io_exit', 'irq_inj']
+    for i in range(len(items)):
+        ret[items[i]] = end_state[i] - start_state[i]
+    return ret