Re: [Autotest] [PATCH] Virt: Adding softlockup subtest

[pradeep <psuriset@linux.vnet.ibm.com>]

On Wed, 20 Jul 2011 22:30:09 -0300
Lucas Meneghel Rodrigues <lmr@redhat.com> wrote:

> From: pradeep <you@example.com>
> 
> This patch introduces a soft lockup/drift test with stress.
> 
>     1) Boot up a VM.
>     2) Build stress on host and guest.
>     3) run heartbeat monitor with the given options on server and
> host. 3) Run for a relatively long time length, ex: 12, 18 or 24
> hours. 4) Output the test result and observe drift.

Thanks for making changes.  
How about taking average of last 10 drift values? 

> 
> Changes from v2:
>  * Fixed up commands being used on guest, lack of proper output
>    redirection was confusing aexpect
>  * Proper clean up previous instances of the monitor programs
>    lying around, as well as log files
>  * Resort to another method of determining host IP if the same
>    has no fully qualified hostname (stand alone laptops, for
>    example)
>  * Only use a single session on guest to execute all the commands.
>    previous version was opening unneeded connections.
>  * Fix stress execution in guest and host, now the stress instances
>    effectively start
>  * Actively open guest and host firewall rules so heartbeat monitor
>    communication can happen
> 
> Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
> Signed-off-by: Pradeep Kumar Surisetty <psuriset@linux.vnet.ibm.com>
> ---
>  client/tests/kvm/deps/heartbeat_slu.py |  205
> ++++++++++++++++++++++++++++++++
> client/tests/kvm/tests_base.cfg.sample |   18 +++
> client/virt/tests/softlockup.py        |  147 +++++++++++++++++++++++
> 3 files changed, 370 insertions(+), 0 deletions(-) create mode 100755
> client/tests/kvm/deps/heartbeat_slu.py create mode 100644
> client/virt/tests/softlockup.py
> 
> diff --git a/client/tests/kvm/deps/heartbeat_slu.py
> b/client/tests/kvm/deps/heartbeat_slu.py new file mode 100755
> index 0000000..697bbbf
> --- /dev/null
> +++ b/client/tests/kvm/deps/heartbeat_slu.py
> @@ -0,0 +1,205 @@
> +#!/usr/bin/env python
> +
> +"""
> +Heartbeat server/client to detect soft lockups
> +"""
> +
> +import socket, os, sys, time, getopt
> +
> +def daemonize(output_file):
> +    try:
> +        pid = os.fork()
> +    except OSError, e:
> +        raise Exception, "error %d: %s" % (e.strerror, e.errno)
> +
> +    if pid:
> +        os._exit(0)
> +
> +    os.umask(0)
> +    os.setsid()
> +    sys.stdout.flush()
> +    sys.stderr.flush()
> +
> +    if file:
> +        output_handle = file(output_file, 'a+', 0)
> +        # autoflush stdout/stderr
> +        sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
> +        sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0)
> +    else:
> +        output_handle = file('/dev/null', 'a+')
> +
> +    stdin_handle = open('/dev/null', 'r')
> +    os.dup2(output_handle.fileno(), sys.stdout.fileno())
> +    os.dup2(output_handle.fileno(), sys.stderr.fileno())
> +    os.dup2(stdin_handle.fileno(), sys.stdin.fileno())
> +
> +def recv_all(sock):
> +    total_data = []
> +    while True:
> +        data = sock.recv(1024)
> +        if not data:
> +            break
> +        total_data.append(data)
> +    return ''.join(total_data)
> +
> +def run_server(host, port, daemon, file, queue_size, threshold,
> drift):
> +    if daemon:
> +        daemonize(output_file=file)
> +    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
> +    sock.bind((host, port))
> +    sock.listen(queue_size)
> +    timeout_interval = threshold * 2
> +    prev_check_timestamp = float(time.time())
> +    while 1:
> +        c_sock, c_addr = sock.accept()
> +        heartbeat = recv_all(c_sock)
> +        local_timestamp = float(time.time())
> +        drift = check_heartbeat(heartbeat, local_timestamp,
> threshold, check_drift)
> +        # NOTE: this doesn't work if the only client is the one that
> timed
> +        # out, but anything more complete would require another
> thread and
> +        # a lock for client_prev_timestamp.
> +        if local_timestamp - prev_check_timestamp > threshold * 2.0:
> +            check_for_timeouts(threshold, check_drift)
> +            prev_check_timestamp = local_timestamp
> +        if verbose:
> +            if check_drift:
> +                print "%.2f: %s (%s)" % (local_timestamp, heartbeat,
> drift)
> +            else:
> +                print "%.2f: %s" % (local_timestamp, heartbeat)
> +
> +def run_client(host, port, daemon, file, interval):
> +    if daemon:
> +        daemonize(output_file=file)
> +    seq = 1
> +    while 1:
> +        try:
> +            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 
> +            sock.connect((host, port))
> +            heartbeat = get_heartbeat(seq)
> +            sock.sendall(heartbeat)
> +            sock.close()
> +            if verbose:
> +                print heartbeat
> +        except socket.error, (value, message):
> +            print "%.2f: ERROR, %d - %s" % (float(time.time()),
> value, message) +
> +        seq += 1
> +        time.sleep(interval)
> +
> +def get_heartbeat(seq=1):
> +    return "%s %06d %.2f" % (hostname, seq, float(time.time()))
> +
> +def check_heartbeat(heartbeat, local_timestamp, threshold,
> check_drift):
> +    hostname, seq, timestamp = heartbeat.rsplit()
> +    timestamp = float(timestamp)
> +    if client_prev_timestamp.has_key(hostname):
> +        delta = local_timestamp - client_prev_timestamp[hostname]
> +        if delta > threshold:
> +            print "%.2f: ALERT, SLU detected on host %s, delta %ds" \
> +                % (float(time.time()), hostname, delta)
> +
> +    client_prev_timestamp[hostname] = local_timestamp
> +
> +    if check_drift:
> +        if not client_clock_offset.has_key(hostname):
> +            client_clock_offset[hostname] = timestamp -
> local_timestamp
> +            client_prev_drift[hostname] = 0
> +        drift = timestamp - local_timestamp -
> client_clock_offset[hostname]
> +        drift_delta = drift - client_prev_drift[hostname]
> +        client_prev_drift[hostname] = drift
> +        return "drift %+4.2f (%+4.2f)" % (drift, drift_delta) 
> +
> +def check_for_timeouts(threshold, check_drift):
> +    local_timestamp = float(time.time())
> +    hostname_list = list(client_prev_timestamp)
> +    for hostname in hostname_list:
> +        timestamp = client_prev_timestamp[hostname]
> +        delta = local_timestamp - timestamp
> +        if delta > threshold * 2:
> +            print "%.2f: ALERT, SLU detected on host %s, no
> heartbeat for %ds" \
> +                % (local_timestamp, hostname, delta)
> +            del client_prev_timestamp[hostname]
> +            if check_drift:
> +                del client_clock_offset[hostname]
> +                del client_prev_drift[hostname]
> +
> +def usage():
> +    print """
> +Usage:
> +    
> +    heartbeat_slu.py --server --address <bind_address> --port
> <bind_port>
> +                     [--file <output_file>] [--no-daemon] [--verbose]
> +                     [--threshold <heartbeat threshold>]
> +
> +    heartbeat_slu.py --client --address <server_address> -p
> <server_port>
> +                     [--file output_file] [--no-daemon] [--verbose]
> +                     [--interval <heartbeat interval in seconds>]
> +"""
> +
> +# host information and global data
> +hostname = socket.gethostname()
> +client_prev_timestamp = {}
> +client_clock_offset = {}
> +client_prev_drift = {}
> +
> +# default param values
> +host_port = 9001
> +host_address = ''
> +interval = 1 # seconds between heartbeats
> +threshold = 10 # seconds late till alert
> +is_server = False
> +is_daemon = True
> +file_server = "/tmp/heartbeat_server.out"
> +file_client = "/tmp/heartbeat_client.out"
> +file_selected = None
> +queue_size = 5
> +verbose = False
> +check_drift = False
> +
> +# process cmdline opts
> +try:
> +    opts, args = getopt.getopt(sys.argv[1:], "vhsfd:p:a:i:t:", [
> +                    "server", "client", "no-daemon", "address=",
> "port=",
> +                    "file=", "server", "interval=", "threshold=",
> "verbose",
> +                    "check-drift", "help"])
> +except getopt.GetoptError, e:
> +    print "error: %s" % str(e)
> +    usage()
> +    exit(1)
> +
> +for param, value in opts:
> +    if param in ["-p", "--port"]:
> +        host_port = int(value)
> +    elif param in ["-a", "--address"]:
> +        host_address = value
> +    elif param in ["-s", "--server"]:
> +        is_server = True
> +    elif param in ["-c", "--client"]:
> +        is_server = False
> +    elif param in ["--no-daemon"]:
> +        is_daemon = False
> +    elif param in ["-f", "--file"]:
> +        file_selected = value
> +    elif param in ["-i", "--interval"]:
> +        interval = int(value)
> +    elif param in ["-t", "--threshold"]:
> +        threshold = int(value)
> +    elif param in ["-d", "--check-drift"]:
> +        check_drift = True
> +    elif param in ["-v", "--verbose"]:
> +        verbose = True
> +    elif param in ["-h", "--help"]:
> +        usage()
> +        exit(0)
> +    else:
> +        print "error: unrecognized option: %s" % value
> +        usage()
> +        exit(1)
> +
> +# run until we're terminated
> +if is_server:
> +    file_server = file_selected or file_server
> +    run_server(host_address, host_port, is_daemon, file_server,
> queue_size, threshold, check_drift) +else:
> +    file_client = file_selected or file_client
> +    run_client(host_address, host_port, is_daemon, file_client,
> interval) diff --git a/client/tests/kvm/tests_base.cfg.sample
> b/client/tests/kvm/tests_base.cfg.sample index 65880d8..e9e41f9 100644
> --- a/client/tests/kvm/tests_base.cfg.sample
> +++ b/client/tests/kvm/tests_base.cfg.sample
> @@ -420,6 +420,24 @@ variants:
>          type = smbios_table
>          start_vm = no
> 
> +    - softlockup: install setup unattended_install.cdrom
> +        only Linux
> +        type = softlockup
> +        softlockup_files = stress-1.0.4.tar.gz
> +        stress_setup_cmd = "cd %s && tar xvf stress-1.0.4.tar.gz &&
> cd stress-1.0.4 && ./configure && make && cd src"
> +        server_setup_cmd = "%s/heartbeat_slu.py --server --threshold
> %s --file %s --port %s --verbose --check-drift"
> +        client_setup_cmd = "%s/heartbeat_slu.py --client --address
> %s --file %s --port %s --interval 1"
> +        stress_cmd  = "cd %s && cd stress-1.0.4 && cd src &&
> nohup ./stress -c %s > /dev/null 2>&1&"
> +        kill_monitor_cmd = "ps aux | grep heart | grep -v grep | awk
> '{print$2}' | xargs kill -9 > /dev/null 2>&1"
> +        kill_stress_cmd = "pkill -f stress > /dev/null 2>&1"
> +        drift_cmd = "tail -1 %s | awk '{print $7}'"
> +        monitor_log_file_server = /tmp/heartbeat_server.log
> +        monitor_log_file_client = /tmp/heartbeat_client.log
> +        monitor_port = 13330
> +        stress_threshold = 10
> +        # time_to_run (hours) = 12, 18, 24, 48 hours
> +        test_length = 0.10
> +
>      - stress_boot: install setup image_copy unattended_install.cdrom
>          type = stress_boot
>          max_vms = 5    
> diff --git a/client/virt/tests/softlockup.py
> b/client/virt/tests/softlockup.py new file mode 100644
> index 0000000..d946965
> --- /dev/null
> +++ b/client/virt/tests/softlockup.py
> @@ -0,0 +1,147 @@
> +import logging, os, socket, time
> +from autotest_lib.client.bin import utils
> +
> +
> +def run_softlockup(test, params, env):
> +    """
> +    soft lockup/drift test with stress.
> +
> +    1) Boot up a VM.
> +    2) Build stress on host and guest.
> +    3) run heartbeat with the given options on server and host.
> +    3) Run for a relatively long time length. ex: 12, 18 or 24 hours.
> +    4) Output the test result and observe drift.
> +
> +    @param test: KVM test object.
> +    @param params: Dictionary with the test parameters.
> +    @param env: Dictionary with test environment.
> +    """
> +    stress_setup_cmd = params.get("stress_setup_cmd")
> +    stress_cmd = params.get("stress_cmd")
> +    server_setup_cmd = params.get("server_setup_cmd")
> +    drift_cmd = params.get("drift_cmd")
> +    kill_stress_cmd = params.get("kill_stress_cmd")
> +    kill_monitor_cmd = params.get("kill_monitor_cmd")
> +
> +    threshold = int(params.get("stress_threshold"))
> +    monitor_log_file_server = params.get("monitor_log_file_server")
> +    monitor_log_file_client = params.get("monitor_log_file_client")
> +    test_length = int(3600 * float(params.get("test_length")))
> +    monitor_port = int(params.get("monitor_port"))
> +
> +    vm = env.get_vm(params["main_vm"])
> +    login_timeout = int(params.get("login_timeout", 360))
> +    stress_dir = os.path.join(os.environ['AUTODIR'], "tests/stress")
> +    monitor_dir = os.path.join(test.bindir, 'deps')
> +
> +
> +    def _kill_guest_programs(session, kill_stress_cmd,
> kill_monitor_cmd):
> +        logging.info("Kill stress and monitor on guest")
> +        try:
> +            session.cmd(kill_stress_cmd)
> +        except:
> +            pass
> +        try:
> +            session.cmd(kill_monitor_cmd)
> +        except:
> +            pass
> +
> +
> +    def _kill_host_programs(kill_stress_cmd, kill_monitor_cmd):
> +        logging.info("Kill stress and monitor on host")
> +        utils.run(kill_stress_cmd, ignore_status=True)
> +        utils.run(kill_monitor_cmd, ignore_status=True)
> +
> +
> +    def host():
> +        logging.info("Setup monitor server on host")
> +        # Kill previous instances of the host load programs, if any
> +        _kill_host_programs(kill_stress_cmd, kill_monitor_cmd)
> +        # Cleanup previous log instances
> +        if os.path.isfile(monitor_log_file_server):
> +            os.remove(monitor_log_file_server)
> +        # Opening firewall ports on host
> +        utils.run("iptables -F", ignore_status=True)
> +
> +        # Run heartbeat on host
> +        utils.run(server_setup_cmd % (monitor_dir, threshold,
> +                                      monitor_log_file_server,
> monitor_port)) +
> +        logging.info("Build stress on host")
> +        # Uncompress and build stress on host
> +        utils.run(stress_setup_cmd % stress_dir)
> +
> +        logging.info("Run stress on host")
> +        # stress_threads = 2 * n_cpus
> +        threads_host = 2 * utils.count_cpus()
> +        # Run stress test on host
> +        utils.run(stress_cmd % (stress_dir, threads_host))
> +
> +
> +    def guest():
> +        try:
> +            host_ip = socket.gethostbyname(socket.gethostname())
> +        except socket.error:
> +            try:
> +                # Hackish, but works well on stand alone (laptop)
> setups
> +                # with access to the internet. If this fails, well,
> then
> +                # not much else can be done...
> +                s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
> +                s.connect(("redhat.com", 80))
> +                host_ip = s.getsockname()[0]
> +            except socket.error, (value, e):
> +                raise error.TestError("Could not determine host IP:
> %d %s" %
> +                                      (value, e))
> +
> +        # Now, starting the guest
> +        vm.verify_alive()
> +        session = vm.wait_for_login(timeout=login_timeout)
> +
> +        # Kill previous instances of the load programs, if any
> +        _kill_guest_programs(session, kill_stress_cmd,
> kill_monitor_cmd)
> +        # Clean up previous log instances
> +        session.cmd("rm -f %s" % monitor_log_file_client)
> +
> +        # Opening firewall ports on guest
> +        try:
> +            session.cmd("iptables -F")
> +        except:
> +            pass
> +
> +        # Get required files and copy them from host to guest
> +        monitor_path = os.path.join(test.bindir, 'deps',
> 'heartbeat_slu.py')
> +        stress_path = os.path.join(os.environ['AUTODIR'], "tests",
> "stress",
> +                                   "stress-1.0.4.tar.gz")
> +        vm.copy_files_to(monitor_path, "/tmp")
> +        vm.copy_files_to(stress_path, "/tmp")
> +
> +        logging.info("Setup monitor client on guest")
> +        # Start heartbeat on guest
> +        session.cmd(params.get("client_setup_cmd") %
> +                    ("/tmp", monitor_log_file_client, host_ip,
> monitor_port)) +

in tests_base.cfg 

client_setup_cmd = "%s/heartbeat_slu.py --client --address %s --file %s
--port %s --interval 1" 

in softlockup.py

        session.cmd(params.get("client_setup_cmd") % ("/tmp",
        monitor_log_file_client, host_ip, 
        monitor_port))

	address, file picking up options interchangeably. 


> +        logging.info("Build stress on guest")
> +        # Uncompress and build stress on guest
> +        session.cmd(stress_setup_cmd % "/tmp", timeout=200)
> +
> +        logging.info("Run stress on guest")
> +        # stress_threads = 2 * n_vcpus
> +        threads_guest = 2 * int(params.get("smp", 1))
> +        # Run stress test on guest
> +        session.cmd(stress_cmd % ("/tmp", threads_guest))
> +
> +        # Wait and report
> +        logging.debug("Wait for %d s", test_length)
> +        time.sleep(test_length)
> +
> +        # Kill instances of the load programs on both guest and host
> +        _kill_guest_programs(session, kill_stress_cmd,
> kill_monitor_cmd)
> +        _kill_host_programs(kill_stress_cmd, kill_monitor_cmd)
> +
> +        # Collect drift
> +        drift = utils.system_output(drift_cmd %
> monitor_log_file_server)
> +        logging.info("Drift noticed: %s", drift)
> +
> +
> +    host()
> +    guest()