Re: [AUTOTEST][PATCH] Add test for testing of killing guest when network is under usage.

["Lukáš Doktor" <ldoktor@redhat.com>]

Hi Jiří,

Do you have any further plans with this test? I'm not convinced that 
netperf only as a stress is necessarily. You can use netcat or simple 
python udp send/recv (flood attack ;-) ).

Dne 17.8.2011 16:17, Jiří Župka napsal(a):
> This patch contain two tests.
> 1) Try kill guest when guest netwok is under loading.
> 2) Try kill guest after multiple adding and removing network drivers.
>
> Signed-off-by: Jiří Župka<jzupka@redhat.com>
> ---
>   client/tests/kvm/tests_base.cfg.sample    |   23 +++++
>   client/virt/tests/netstress_kill_guest.py |  146 +++++++++++++++++++++++++++++
>   2 files changed, 169 insertions(+), 0 deletions(-)
>   create mode 100644 client/virt/tests/netstress_kill_guest.py
>
> diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample
> index ec1b48d..2c88088 100644
> --- a/client/tests/kvm/tests_base.cfg.sample
> +++ b/client/tests/kvm/tests_base.cfg.sample
> @@ -845,6 +845,29 @@ variants:
>           restart_vm = yes
>           kill_vm_on_error = yes
>
> +    - netstress_kill_guest: install setup unattended_install.cdrom
> +        only Linux
> +        type = netstress_kill_guest
> +        image_snapshot = yes
> +        nic_mode = tap
> +        # There should be enough vms for build topology.
> +        variants:
> +            -driver:
> +                mode = driver
> +            -load:
> +                mode = load
> +                netperf_files = netperf-2.4.5.tar.bz2 wait_before_data.patch
> +                packet_size = 1500
> +                setup_cmd = "cd %s&&  tar xvfj netperf-2.4.5.tar.bz2&&  cd netperf-2.4.5&&  patch -p0<  ../wait_before_data.patch&&  ./configure&&  make"
> +                clean_cmd = " while killall -9 netserver; do True test; done;"
> +                netserver_cmd =  %s/netperf-2.4.5/src/netserver
> +                netperf_cmd = %s/netperf-2.4.5/src/netperf -t %s -H %s -l 60 -- -m %s
> +        variants:
> +            - vhost:
> +                netdev_extra_params = "vhost=on"


You might add "modprobe vhost-net" command as vhost-net might not be 
loaded by default.

> +            - vhost-no:
> +                netdev_extra_params = ""
> +
>       - set_link: install setup image_copy unattended_install.cdrom
>           type = set_link
>           test_timeout = 1000
> diff --git a/client/virt/tests/netstress_kill_guest.py b/client/virt/tests/netstress_kill_guest.py
> new file mode 100644
> index 0000000..7daec95
> --- /dev/null
> +++ b/client/virt/tests/netstress_kill_guest.py
> @@ -0,0 +1,146 @@
> +import logging, os, signal, re, time
> +from autotest_lib.client.common_lib import error
> +from autotest_lib.client.bin import utils
> +from autotest_lib.client.virt import aexpect, virt_utils
> +
> +
> +def run_netstress_kill_guest(test, params, env):
> +    """
> +    Try stop network interface in VM when other VM try to communicate.
> +
> +    @param test: kvm test object
> +    @param params: Dictionary with the test parameters
> +    @param env: Dictionary with test environment.
> +    """
> +    def get_corespond_ip(ip):
> +        """
> +        Get local ip address which is used for contact ip.
> +
> +        @param ip: Remote ip
> +        @return: Local corespond IP.
> +        """
> +        result = utils.run("ip route get %s" % (ip)).stdout
> +        ip = re.search("src (.+)", result)
> +        if ip is not None:
> +            ip = ip.groups()[0]
> +        return ip
> +
> +
> +    def get_ethernet_driver(session):
> +        """
> +        Get driver of network cards.
> +
> +        @param session: session to machine
> +        """
> +        modules = []
> +        out = session.cmd("ls -l /sys/class/net/*/device/driver/module")
> +        for module in out.split("\n"):
> +            modules.append(module.split("/")[-1])
> +        modules.remove("")
> +        return set(modules)
> +
> +
> +    def kill_and_check(vm):
> +        vm_pid = vm.get_pid()
> +        vm.destroy(gracefully=False)
> +        time.sleep(2)
> +        try:
> +            os.kill(vm_pid, 0)
> +            logging.error("VM is not dead.")
> +            raise error.TestFail("Problem with killing guest.")
> +        except OSError:
> +            logging.info("VM is dead.")
> +
> +
> +    def netload_kill_problem(session_serial):

I think you should clean this function. I belive it would be better and 
more readable, if you first get all the params/variables, than prepare 
the host/guests and after all of this start the guest. See the comments 
further...

> +        netperf_dir = os.path.join(os.environ['AUTODIR'], "tests/netperf2")
> +        setup_cmd = params.get("setup_cmd")
> +        clean_cmd = params.get("clean_cmd")
> +
> +        firewall_flush = "iptables -F"
> +        session_serial.cmd_output(firewall_flush)
> +        try:
> +            utils.run("iptables -F")
you have firewall_flush command-string, why not to use it here to. Also 
you should either warn everywhere or not at all... (you log the failure 
when flushing the guest but not here)

> +        except:
> +            pass
> +
> +        for i in params.get("netperf_files").split():
> +            vm.copy_files_to(os.path.join(netperf_dir, i), "/tmp")
> +
> +        try:
> +            session_serial.cmd(firewall_flush)
> +        except aexpect.ShellError:
> +            logging.warning("Could not flush firewall rules on guest")
> +
> +        guest_ip = vm.get_address(0)
> +        server_ip = get_corespond_ip(guest_ip)
> +
> +        session_serial.cmd(setup_cmd % "/tmp", timeout=200)
> +        try:
> +            session_serial.cmd(clean_cmd)
> +        except:
> +            pass
> +        session_serial.cmd(params.get("netserver_cmd") % "/tmp")
> +
> +        utils.run(clean_cmd, ignore_status=True)
> +        utils.run(params.get("netserver_cmd") % netperf_dir)
prior to this command you should prepare the netperf

utils.run(setup_cmd % netperf_dir)

You have it further in the source (perhaps you have it already built on 
your system thus it works for you).

> +
> +        server_netperf_cmd = params.get("netperf_cmd") % (netperf_dir, "TCP_STREAM",
> +                                        guest_ip, params.get("packet_size", "1500"))
> +        quest_netperf_cmd = params.get("netperf_cmd") % ("/tmp", "TCP_STREAM",
> +                                       server_ip, params.get("packet_size", "1500"))
> +
> +        tcpdump = env.get("tcpdump")
> +        pid = None
> +        if tcpdump:
> +            # Stop the background tcpdump process
> +            try:
> +                pid = int(utils.system_output("pidof tcpdump"))
> +                logging.debug("Stopping the background tcpdump")
> +                os.kill(pid, signal.SIGSTOP)
> +            except:
> +                pass
> +
> +        try:
> +            logging.info("Setup and run netperf clients on host")
> +            utils.run(setup_cmd % netperf_dir)
This won't be needed once you execute this at the beginning. (see the 
comment before)

> +
> +            logging.info("Start heavy network load host<=>  guest.")
> +            session_serial.sendline(quest_netperf_cmd)
> +            utils.BgJob(server_netperf_cmd)
> +
> +            #Wait for create big network usage.
> +            time.sleep(10)
> +            kill_and_check(vm)
> +
> +        finally:
> +            utils.run(clean_cmd, ignore_status=True)
> +            if tcpdump and pid:
> +                logging.debug("Resuming the background tcpdump")
> +                logging.info("pid is %s" % pid)
> +                os.kill(pid, signal.SIGCONT)
> +
> +    def netdriver_kill_problem(session_serial):
> +        modules = get_ethernet_driver(session_serial)
> +        print modules
> +        for _ in range(50):
> +            for module in modules:
> +                session_serial.cmd("rmmod %s" % (module))
> +                time.sleep(0.2)
> +            for module in modules:
> +                session_serial.cmd("modprobe %s" % (module))
> +                time.sleep(0.2)
> +        kill_and_check(vm)
> +
> +    vm = env.get_vm(params["main_vm"])
> +    vm.verify_alive()
> +    login_timeout = int(params.get("login_timeout", 360))
> +    session = vm.wait_for_login(timeout=login_timeout)
> +    session.close()
> +    session_serial = vm.wait_for_serial_login(timeout=login_timeout)
> +
> +    mode = params.get("mode")
> +    if mode == "driver":
> +        netdriver_kill_problem(session_serial)
> +    elif mode == "load":
> +        netload_kill_problem(session_serial)


The "driver" test is good. The load test with the netperf setup 
modification works also fine. Would you please clean the function and 
resend the patch and clarify why you use such a heavy tool for simple 
stress test?

Thanks,
Lukáš