[KVM_AUTOTEST PATCH] Added functionality to the preprocessor to run scripts

[KVM_AUTOTEST PATCH] Added functionality to the preprocessor to run scripts

[David Huff]

This patch adds the processes_command function to the preprocessor which
will run custom scripts before and after a test is executed.

processes_command will export all test parameters as environmental variables
for passing parameters to a custom script.

processes_command uses parameters defined in the config file:
pre_command, pre_command_timeout, pre_command_noncritical
post_command, post_command_timeout, post_commmand_noncritical

command - just defines the script to be run
timeout - timeout to kill script if hung
noncritical - if yes do not fail test if command fails, default no

Currently it is still using the returned status from kvm_utils.run_bg If
kvm_utils.run_bg is changed to throw an exception, process_command will
have to be updated at that time.
---
 client/tests/kvm/kvm_preprocessing.py |   48 +++++++++++++++++++++++++++++++-
 1 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/client/tests/kvm/kvm_preprocessing.py b/client/tests/kvm/kvm_preprocessing.py
index f60cfe8..c74d0d9 100644
--- a/client/tests/kvm/kvm_preprocessing.py
+++ b/client/tests/kvm/kvm_preprocessing.py
@@ -1,4 +1,4 @@
-import sys, os, time, commands, re, logging
+import sys, os, time, commands, re, logging, signal
 from autotest_lib.client.bin import test
 from autotest_lib.client.common_lib import error
 import kvm_vm, kvm_utils
@@ -133,6 +133,40 @@ def postprocess_vm(test, params, env, name):
         vm.destroy(gracefully = params.get("kill_vm_gracefully") == "yes")
 
 
+def process_command(test, params, env, command, command_timeout,
+                    command_noncritical):
+    """
+    Pre- or post- custom commands to be executed before/after a test is run 
+    
+    @param test: An Autotest test object.
+    @param params: A dict containing all VM and image parameters.
+    @param env: The environment (a dict-like object).
+    @param command: Script containing the command to be run.
+    @param commmand_timeout: Timeout for command execution.
+    @param command_noncritical: if 'yes' test will not fail if command fails.
+    """
+    if command_timeout is None:
+        command_timeout = "600"
+        
+    if command_noncritical is None:
+        command_noncritical = "no"    
+
+    # export environment vars
+    for k in params.keys():
+        logging.info("Adding KVM_TEST_%s to Environment" % (k))
+        os.putenv("KVM_TEST_%s" % (k), str(params[k]))
+    # execute command
+    logging.info("Executing command '%s'..." % command)
+    timeout = int(command_timeout)
+    (status, pid, output) = kvm_utils.run_bg("cd %s; %s" % (test.bindir, command),
+                                             None, logging.debug, "(command) ", timeout = timeout)
+    if status != 0:
+        kvm_utils.safe_kill(pid, signal.SIGTERM)
+        logging.warn("Custom processing command failed: '%s'..." % command)
+        if command_noncritical != "yes":
+            raise error.TestError, "Custom processing command failed"
+
+
 def process(test, params, env, image_func, vm_func):
     """
     Pre- or post-process VMs and images according to the instructions in params.
@@ -190,6 +224,11 @@ def preprocess(test, params, env):
             vm.destroy()
             del env[key]
 
+    #execute any pre_commands
+    if params.get("pre_command"):
+       process_command(test, params, env, params.get("pre_command"), params.get("pre_command_timeout"),
+               params.get("pre_command_noncritical"))      
+
     # Preprocess all VMs and images
     process(test, params, env, preprocess_image, preprocess_vm)
 
@@ -240,7 +279,12 @@ def postprocess(test, params, env):
         logging.debug("'keep_ppm_files' not specified; removing all PPM files"
                       " from results dir...")
         kvm_utils.run_bg("rm -vf %s" % os.path.join(test.debugdir, "*.ppm"),
-                          None, logging.debug, "(rm) ", timeout=5.0)
+                          None, logging.debug, "(rm) ", timeout = 5.0)
+
+    #execute any post_commands
+    if params.get("post_command"):
+       process_command(test, params, env, params.get("post_command"), params.get("post_command_timeout"),
+               params.get("post_command_noncritical"))
 
 
 def postprocess_on_error(test, params, env):
-- 
1.6.0.6

Re: [KVM_AUTOTEST PATCH] Added functionality to the preprocessor to run scripts

[Lucas Meneghel Rodrigues]

On Fri, 2009-06-12 at 10:43 -0400, David Huff wrote:
> This patch adds the processes_command function to the preprocessor which
> will run custom scripts before and after a test is executed.
> 
> processes_command will export all test parameters as environmental variables
> for passing parameters to a custom script.
> 
> processes_command uses parameters defined in the config file:
> pre_command, pre_command_timeout, pre_command_noncritical
> post_command, post_command_timeout, post_commmand_noncritical
> 
> command - just defines the script to be run
> timeout - timeout to kill script if hung
> noncritical - if yes do not fail test if command fails, default no
> 
> Currently it is still using the returned status from kvm_utils.run_bg If
> kvm_utils.run_bg is changed to throw an exception, process_command will
> have to be updated at that time.

Ok David, I have applied your patch with some slight modifications.
Please check it out. Thanks!

Index: client/tests/kvm/kvm_preprocessing.py
===================================================================
--- client/tests/kvm/kvm_preprocessing.py	(revision 3261)
+++ client/tests/kvm/kvm_preprocessing.py	(working copy)
@@ -1,4 +1,4 @@
-import sys, os, time, commands, re, logging
+import sys, os, time, commands, re, logging, signal
 from autotest_lib.client.bin import test
 from autotest_lib.client.common_lib import error
 import kvm_vm, kvm_utils
@@ -133,6 +133,43 @@
         vm.destroy(gracefully = params.get("kill_vm_gracefully") == "yes")
 
 
+def process_command(test, params, env, command, command_timeout,
+                    command_noncritical):
+    """
+    Pre- or post- custom commands to be executed before/after a test is run
+
+    @param test: An Autotest test object.
+    @param params: A dict containing all VM and image parameters.
+    @param env: The environment (a dict-like object).
+    @param command: Script containing the command to be run.
+    @param commmand_timeout: Timeout for command execution.
+    @param command_noncritical: if 'yes' test will not fail if command fails.
+    """
+    if command_timeout is None:
+        command_timeout = "600"
+
+    if command_noncritical is None:
+        command_noncritical = "no"
+
+    # export environment vars
+    for k in params.keys():
+        logging.info("Adding KVM_TEST_%s to Environment" % (k))
+        os.putenv("KVM_TEST_%s" % (k), str(params[k]))
+    # execute command
+    logging.info("Executing command '%s'..." % command)
+    timeout = int(command_timeout)
+    (status, pid, output) = kvm_utils.run_bg("cd %s; %s" % (test.bindir,
+                                                            command),
+                                                            None, logging.debug,
+                                                            "(command) ",
+                                                            timeout = timeout)
+    if status != 0:
+        kvm_utils.safe_kill(pid, signal.SIGTERM)
+        logging.warn("Custom processing command failed: '%s'..." % command)
+        if command_noncritical != "yes":
+            raise error.TestError("Custom processing command failed")
+
+
 def process(test, params, env, image_func, vm_func):
     """
     Pre- or post-process VMs and images according to the instructions in params.
@@ -190,6 +227,12 @@
             vm.destroy()
             del env[key]
 
+    #execute any pre_commands
+    if params.get("pre_command"):
+        process_command(test, params, env, params.get("pre_command"),
+                        params.get("pre_command_timeout"),
+                        params.get("pre_command_noncritical"))
+
     # Preprocess all VMs and images
     process(test, params, env, preprocess_image, preprocess_vm)
 
@@ -242,7 +285,13 @@
         kvm_utils.run_bg("rm -vf %s" % os.path.join(test.debugdir, "*.ppm"),
                           None, logging.debug, "(rm) ", timeout=5.0)
 
+    #execute any post_commands
+    if params.get("post_command"):
+        process_command(test, params, env, params.get("post_command"),
+                        params.get("post_command_timeout"),
+                        params.get("post_command_noncritical"))
 
+
 def postprocess_on_error(test, params, env):
     """
     Perform postprocessing operations required only if the test failed.

-- 
Lucas Meneghel Rodrigues
Software Engineer (QE)
Red Hat - Emerging Technologies

Re: [KVM_AUTOTEST PATCH] Added functionality to the preprocessor to run scripts

[David Huff]

Lucas Meneghel Rodrigues wrote:
> On Fri, 2009-06-12 at 10:43 -0400, David Huff wrote:
>> This patch adds the processes_command function to the preprocessor which
>> will run custom scripts before and after a test is executed.
>>
>> processes_command will export all test parameters as environmental variables
>> for passing parameters to a custom script.
>>
>> processes_command uses parameters defined in the config file:
>> pre_command, pre_command_timeout, pre_command_noncritical
>> post_command, post_command_timeout, post_commmand_noncritical
>>
>> command - just defines the script to be run
>> timeout - timeout to kill script if hung
>> noncritical - if yes do not fail test if command fails, default no
>>
>> Currently it is still using the returned status from kvm_utils.run_bg If
>> kvm_utils.run_bg is changed to throw an exception, process_command will
>> have to be updated at that time.
> 
> Ok David, I have applied your patch with some slight modifications.
> Please check it out. Thanks!
> 
> Index: client/tests/kvm/kvm_preprocessing.py
> ===================================================================
> --- client/tests/kvm/kvm_preprocessing.py	(revision 3261)
> +++ client/tests/kvm/kvm_preprocessing.py	(working copy)
> @@ -1,4 +1,4 @@
> -import sys, os, time, commands, re, logging
> +import sys, os, time, commands, re, logging, signal
>  from autotest_lib.client.bin import test
>  from autotest_lib.client.common_lib import error
>  import kvm_vm, kvm_utils
> @@ -133,6 +133,43 @@
>          vm.destroy(gracefully = params.get("kill_vm_gracefully") == "yes")
>  
>  
> +def process_command(test, params, env, command, command_timeout,
> +                    command_noncritical):
> +    """
> +    Pre- or post- custom commands to be executed before/after a test is run
> +
> +    @param test: An Autotest test object.
> +    @param params: A dict containing all VM and image parameters.
> +    @param env: The environment (a dict-like object).
> +    @param command: Script containing the command to be run.
> +    @param commmand_timeout: Timeout for command execution.
> +    @param command_noncritical: if 'yes' test will not fail if command fails.
> +    """
> +    if command_timeout is None:
> +        command_timeout = "600"
> +
> +    if command_noncritical is None:
> +        command_noncritical = "no"
> +
> +    # export environment vars
> +    for k in params.keys():
> +        logging.info("Adding KVM_TEST_%s to Environment" % (k))

This was changed to logging.debug per Uri's request

> +        os.putenv("KVM_TEST_%s" % (k), str(params[k]))
> +    # execute command
> +    logging.info("Executing command '%s'..." % command)
> +    timeout = int(command_timeout)
> +    (status, pid, output) = kvm_utils.run_bg("cd %s; %s" % (test.bindir,
> +                                                            command),
> +                                                            None, logging.debug,
> +                                                            "(command) ",

This was also changed to  "(%s) " % os.path.basename(command), per Uri's
comments....

> +                                                            timeout = timeout)
> +    if status != 0:
> +        kvm_utils.safe_kill(pid, signal.SIGTERM)
> +        logging.warn("Custom processing command failed: '%s'..." % command)
> +        if command_noncritical != "yes":
> +            raise error.TestError("Custom processing command failed")

I also changed the verbiage here for better clarification however not
critical...

> +
> +
>  def process(test, params, env, image_func, vm_func):
>      """
>      Pre- or post-process VMs and images according to the instructions in params.
> @@ -190,6 +227,12 @@
>              vm.destroy()
>              del env[key]
>  
> +    #execute any pre_commands
> +    if params.get("pre_command"):
> +        process_command(test, params, env, params.get("pre_command"),
> +                        params.get("pre_command_timeout"),
> +                        params.get("pre_command_noncritical"))
> +
>      # Preprocess all VMs and images
>      process(test, params, env, preprocess_image, preprocess_vm)
>  
> @@ -242,7 +285,13 @@
>          kvm_utils.run_bg("rm -vf %s" % os.path.join(test.debugdir, "*.ppm"),
>                            None, logging.debug, "(rm) ", timeout=5.0)
>  
> +    #execute any post_commands
> +    if params.get("post_command"):
> +        process_command(test, params, env, params.get("post_command"),
> +                        params.get("post_command_timeout"),
> +                        params.get("post_command_noncritical"))
>  
> +
>  def postprocess_on_error(test, params, env):
>      """
>      Perform postprocessing operations required only if the test failed.
>