Re: [PATCH] KVM test: Add a subtest kdump

[Lucas Meneghel Rodrigues <lmr@redhat.com>]

On Thu, 2010-10-28 at 15:36 +0800, Jason Wang wrote:
> Add a new subtest to check whether kdump work correctly in guest. This test just
> try to trigger crash on each vcpu and then verify it by checking the vmcore.

Nice test Jason, some comments below:

> Signed-off-by: Jason Wang <jasowang@redhat.com>
> ---
>  client/tests/kvm/tests/kdump.py              |   79 ++++++++++++++++++++++++++
>  client/tests/kvm/tests_base.cfg.sample       |   11 ++++
>  client/tests/kvm/unattended/RHEL-5-series.ks |    1 
>  3 files changed, 91 insertions(+), 0 deletions(-)
>  create mode 100644 client/tests/kvm/tests/kdump.py
> 
> diff --git a/client/tests/kvm/tests/kdump.py b/client/tests/kvm/tests/kdump.py
> new file mode 100644
> index 0000000..8fa3cca
> --- /dev/null
> +++ b/client/tests/kvm/tests/kdump.py
> @@ -0,0 +1,79 @@
> +import logging, time
> +from autotest_lib.client.common_lib import error
> +import kvm_subprocess, kvm_test_utils, kvm_utils
> +
> +
> +def run_kdump(test, params, env):
> +    """
> +    KVM reboot test:
> +    1) Log into a guest
> +    2) Check and enable the kdump
> +    3) For each vcpu, trigger a crash and check the vmcore
> +
> +    @param test: kvm test object
> +    @param params: Dictionary with the test parameters
> +    @param env: Dictionary with test environment.
> +    """
> +    vm = kvm_test_utils.get_living_vm(env, params.get("main_vm"))
> +    timeout = float(params.get("login_timeout", 240))
> +    crash_timeout = float(params.get("crash_timeout", 360))
> +    session = kvm_test_utils.wait_for_login(vm, 0, timeout, 0, 2)
> +    def_kernel_param_cmd = "grubby --update-kernel=`grubby --default-kernel`" \
> +        " --args=crashkernel=128M@64M"

^ Implicit line continuation is better here

def_kernel_param_cmd = ("command param1 param 2..."
                        "param8 param9")

> +    kernel_param_cmd = params.get("kernel_param_cmd", def_kernel_param_cmd)
> +    def_kdump_enable_cmd = "chkconfig kdump on && service kdump start"
> +    kdump_enable_cmd = params.get("kdump_enable_cmd", def_kdump_enable_cmd)
> +
> +    def crash_test(vcpu):
> +        """
> +        Trigger a crash dump through sysrq-trigger
> +
> +        @param vcpu: vcpu which is used to trigger a crash
> +        """
> +        session = kvm_test_utils.wait_for_login(vm, 0, timeout, 0, 2)
> +        session.get_command_status("rm -rf /var/crash/*")
> +
> +        logging.info("Triggering crash on vcpu %d ...", vcpu)
> +        crash_cmd = "taskset -c %d echo c > /proc/sysrq-trigger" % vcpu
> +        session.sendline(crash_cmd)
> +
> +        if not kvm_utils.wait_for(lambda: not session.is_responsive(), 240, 0,
> +                                  1):
> +            raise error.TestFail("Could not trigger crash on vcpu %d" % vcpu)
> +
> +        logging.info("Waiting for the completion of dumping")

^ "Waiting for kernel crash dump to complete" would be better

> +        session = kvm_test_utils.wait_for_login(vm, 0, crash_timeout, 0, 2)
> +
> +        logging.info("Probing vmcore file ...")
> +        s = session.get_command_status("ls -R /var/crash | grep vmcore")
> +        if s != 0:
> +            raise error.TestFail("Could not find the generated vmcore file!")
> +        else:
> +            logging.info("Found vmcore.")
> +
> +        session.get_command_status("rm -rf /var/crash/*")
> +
> +    try:
> +        logging.info("Check the existence of crash kernel ...")
> +        prob_cmd = "grep -q 1 /sys/kernel/kexec_crash_loaded"
> +        s = session.get_command_status(prob_cmd)
> +        if s != 0:
> +            logging.info("Crash kernel is not loaded. Try to load it.")
> +            # We need to setup the kernel params
> +            s, o = session.get_command_status_output(kernel_param_cmd)
> +            if s != 0:
> +                raise error.TestFail("Could not add crashkernel params to"
> +                                     "kernel")
> +            session = kvm_test_utils.reboot(vm, session, timeout=timeout);
> +
> +        logging.info("Enable kdump service ...")
> +        # the initrd may be rebuilt here so we need to wait a little more
> +        s, o = session.get_command_status_output(kdump_enable_cmd, timeout=120)

^ I remember initrd built usually takes longer than 2 minutes in most
machines, does this work fine on both Fedora and RHEL?

> +        if s != 0:
> +            raise error.TestFail("Could not enable kdump service:%s" % o)
> +
> +        nvcpu = int(params.get("smp", 1))
> +        [crash_test(i) for i in range(nvcpu)]

^ Although list comprehension is indeed very cool, since we're not going
to do anything with this list, I'd rather prefer to use the good old for
loop.

> +    finally:
> +        session.close()
> diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample
> index fe3563c..25ad688 100644
> --- a/client/tests/kvm/tests_base.cfg.sample
> +++ b/client/tests/kvm/tests_base.cfg.sample
> @@ -665,6 +665,15 @@ variants:
>                  image_name_snapshot1 = sn1
>                  image_name_snapshot2 = sn2
>  
> +    - kdump:
> +        type = kdump
> +        # time waited for the completion of crash dump
> +        # crash_timeout = 360
> +        # command to add the crashkernel=X@Y to kernel cmd line
> +        # kernel_param_cmd = "grubby --update-kernel=`grubby --default-kernel` --args=crashkernel=128M@64M"
> +        # command to enable kdump service
> +        # kdump_enable_cmd = chkconfig kdump on && service kdump start
> +
>      # system_powerdown, system_reset and shutdown *must* be the last ones
>      # defined (in this order), since the effect of such tests can leave
>      # the VM on a bad state.
> @@ -1924,6 +1933,8 @@ virtio_net|virtio_blk|e1000|balloon_check:
>      only Fedora.11 Fedora.12 Fedora.13 RHEL.5 OpenSUSE.11 SLES.11 Ubuntu-8.10-server
>      # only WinXP Win2003 Win2008 WinVista Win7 Fedora.11 Fedora.12 Fedora.13 RHEL.5 OpenSUSE.11 SLES.11 Ubuntu-8.10-server
>  
> +kdump:
> +    only Fedora RHEL.5
>  
>  variants:
>      - @qcow2:
> diff --git a/client/tests/kvm/unattended/RHEL-5-series.ks b/client/tests/kvm/unattended/RHEL-5-series.ks
> index 92ff727..3ee84f1 100644
> --- a/client/tests/kvm/unattended/RHEL-5-series.ks
> +++ b/client/tests/kvm/unattended/RHEL-5-series.ks

^ Is kexec-tools installed on our Fedora kickstarts, no need to add it
just the way you did with RHEL5? If not, the Fedora kickstart files need
to be patched as well.

> @@ -21,6 +21,7 @@ reboot
>  @base
>  @development-libs
>  @development-tools
> +kexec-tools
>  
>  %post --interpreter /usr/bin/python
>  import socket, os
> 
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html