[KVM-AUTOTEST PATCH] KVM test: Add hugepage variant

[KVM-AUTOTEST PATCH] KVM test: Add hugepage variant

[Lucas Meneghel Rodrigues]

This patch adds a small setup script to set up huge memory
pages during the kvm tests execution. Also, added hugepage setup to the
fc8_quick sample.

Signed-off-by: Lukáš Doktor <ldoktor@redhat.com>
Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/kvm_tests.cfg.sample |    8 +++
 client/tests/kvm/kvm_vm.py            |   11 +++
 client/tests/kvm/scripts/hugepage.py  |  109 +++++++++++++++++++++++++++++++++
 3 files changed, 128 insertions(+), 0 deletions(-)
 create mode 100644 client/tests/kvm/scripts/hugepage.py

diff --git a/client/tests/kvm/kvm_tests.cfg.sample b/client/tests/kvm/kvm_tests.cfg.sample
index 2d75a66..7cd12cb 100644
--- a/client/tests/kvm/kvm_tests.cfg.sample
+++ b/client/tests/kvm/kvm_tests.cfg.sample
@@ -587,6 +587,13 @@ variants:
 
 
 variants:
+    - @kvm_smallpages:
+    - kvm_hugepages:
+        pre_command = "/usr/bin/python scripts/hugepage.py /mnt/kvm_hugepage"
+        extra_params += " -mem-path /mnt/kvm_hugepage"
+
+
+variants:
     - @basic:
         only Fedora Windows
     - @full:
@@ -598,6 +605,7 @@ variants:
         only Fedora.8.32
         only install setup boot shutdown
         only rtl8139
+        only kvm_hugepages
     - @sample1:
         only qcow2
         only ide
diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py
index d96b359..eba9b84 100644
--- a/client/tests/kvm/kvm_vm.py
+++ b/client/tests/kvm/kvm_vm.py
@@ -397,6 +397,17 @@ class VM:
                 self.destroy()
                 return False
 
+            # Get the output so far, to see if we have any problems with
+            # hugepage setup.
+            output = self.process.get_output()
+
+            if "alloc_mem_area" in output:
+                logging.error("Could not allocate hugepage memory; "
+                              "qemu command:\n%s" % qemu_command)
+                logging.error("Output:" + kvm_utils.format_str_for_message(
+                              self.process.get_output()))
+                return False
+
             logging.debug("VM appears to be alive with PID %d",
                           self.process.get_pid())
             return True
diff --git a/client/tests/kvm/scripts/hugepage.py b/client/tests/kvm/scripts/hugepage.py
new file mode 100644
index 0000000..dc36da4
--- /dev/null
+++ b/client/tests/kvm/scripts/hugepage.py
@@ -0,0 +1,109 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+import os, sys, time
+
+"""
+Simple script to allocate enough hugepages for KVM testing purposes.
+"""
+
+class HugePageError(Exception):
+    """
+    Simple wrapper for the builtin Exception class.
+    """
+    pass
+
+
+class HugePage:
+    def __init__(self, hugepage_path=None):
+        """
+        Gets environment variable values and calculates the target number
+        of huge memory pages.
+
+        @param hugepage_path: Path where to mount hugetlbfs path, if not
+                yet configured.
+        """
+        self.vms = len(os.environ['KVM_TEST_vms'].split())
+        self.mem = int(os.environ['KVM_TEST_mem'])
+        try:
+            self.max_vms = int(os.environ['KVM_TEST_max_vms'])
+        except KeyError:
+            self.max_vms = 0
+
+        if hugepage_path:
+            self.hugepage_path = hugepage_path
+        else:
+            self.hugepage_path = '/mnt/kvm_hugepage'
+
+        self.hugepage_size = self.get_hugepage_size()
+        self.target_hugepages = self.get_target_hugepages()
+
+
+    def get_hugepage_size(self):
+        """
+        Get the current system setting for huge memory page size.
+        """
+        meminfo = open('/proc/meminfo', 'r').readlines()
+        huge_line_list = [h for h in meminfo if h.startswith("Hugepagesize")]
+        try:
+            return int(huge_line_list[0].split()[1])
+        except ValueError, e:
+            raise HugePageError("Could not get huge page size setting from "
+                                "/proc/meminfo: %s" % e)
+
+
+    def get_target_hugepages(self):
+        """
+        Calculate the target number of hugepages for testing purposes.
+        """
+        if self.vms < self.max_vms:
+            self.vms = self.max_vms
+        vmsm = (self.vms * self.mem) + (self.vms * 64)
+        return int(vmsm * 1024 / self.hugepage_size)
+
+
+    def set_hugepages(self):
+        """
+        Sets the hugepage limit to the target hugepage value calculated.
+        """
+        hugepage_cfg = open("/proc/sys/vm/nr_hugepages", "r+")
+        hp = hugepage_cfg.readline()
+        while int(hp) < self.target_hugepages:
+            loop_hp = hp
+            hugepage_cfg.write(str(self.target_hugepages))
+            hugepage_cfg.flush()
+            hugepage_cfg.seek(0)
+            hp = int(hugepage_cfg.readline())
+            if loop_hp == hp:
+                raise HugePageError("Cannot set the kernel hugepage setting "
+                                    "to the target value of %d hugepages." %
+                                    self.target_hugepages)
+        hugepage_cfg.close()
+
+
+    def mount_hugepage_fs(self):
+        """
+        Verify if there's a hugetlbfs mount set. If there's none, will set up
+        a hugetlbfs mount using the class attribute that defines the mount
+        point.
+        """
+        if not os.path.ismount(self.hugepage_path):
+            if not os.path.isdir(self.hugepage_path):
+                os.makedirs(self.hugepage_path)
+            cmd = "mount -t hugetlbfs none %s" % self.hugepage_path
+            if os.system(cmd):
+                raise HugePageError("Cannot mount hugetlbfs path %s" %
+                                    self.hugepage_path)
+
+
+    def setup(self):
+        self.set_hugepages()
+        self.mount_hugepage_fs()
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        huge_page = HugePage()
+    else:
+        huge_page = HugePage(sys.argv[1])
+
+    huge_page.setup()
-- 
1.6.2.5

[KVM AUTOTEST PATCH] [RFC] KVM test: keep record of supported qemu options

[Lucas Meneghel Rodrigues]

In order to make it easier to figure out problems and
also to avoid aborting tests prematurely due to
incompatible qemu options, keep record of supported
qemu options, and if extra options are passed to qemu,
verify if they are amongst the supported options. Also,
try to replace known misspelings on options in case
something goes wrong, and be generous logging any problems.

This first version of the patch gets supported flags from
the output of qemu --help. I thought this would be good
enough for a first start. I am asking for input on whether
this is needed, and if yes, if the approach looks good.

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/kvm_vm.py |   79 ++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 77 insertions(+), 2 deletions(-)

diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py
index eba9b84..0dd34c2 100644
--- a/client/tests/kvm/kvm_vm.py
+++ b/client/tests/kvm/kvm_vm.py
@@ -121,6 +121,7 @@ class VM:
         self.qemu_path = qemu_path
         self.image_dir = image_dir
         self.iso_dir = iso_dir
+        self.qemu_supported_flags = self.get_qemu_supported_flags()
 
 
         # Find available monitor filename
@@ -258,7 +259,7 @@ class VM:
 
         extra_params = params.get("extra_params")
         if extra_params:
-            qemu_cmd += " %s" % extra_params
+            qemu_cmd += " %s" % self.process_qemu_extra_params(extra_params)
 
         for redir_name in kvm_utils.get_sub_dict_names(params, "redirs"):
             redir_params = kvm_utils.get_sub_dict(params, redir_name)
@@ -751,7 +752,7 @@ class VM:
             else:
                 self.send_key(char)
 
-    
+
     def get_uuid(self):
         """
         Catch UUID of the VM.
@@ -762,3 +763,77 @@ class VM:
             return self.uuid
         else:
             return self.params.get("uuid", None)
+
+
+    def get_qemu_supported_flags(self):
+        """
+        Gets all supported qemu options from qemu-help. This is a useful
+        procedure to quickly spot problems with incompatible qemu flags.
+        """
+        cmd = self.qemu_path + ' --help'
+        (status, output) = kvm_subprocess.run_fg(cmd)
+        supported_flags = []
+
+        if status:
+            logging.error('Process qemu --help ended with exit code !=0. '
+                          'No supported qemu flags will be recorded.')
+            return supported_flags
+
+        for line in output.split('\n'):
+            if line and line.startswith('-'):
+                flag = line.split()[0]
+                if flag not in supported_flags:
+                    supported_flags.append(flag)
+
+        return supported_flags
+
+
+    def process_qemu_extra_params(self, extra_params):
+        """
+        Verifies an extra param passed to qemu to see if it's supported by the
+        current qemu version. If it's not supported, try to find an appropriate
+        replacement on a list of known option misspellings.
+
+        @param extra_params: String with a qemu command line option.
+        """
+        flag = extra_params.split()[0]
+
+        if flag not in self.qemu_supported_flags:
+            logging.error("Flag %s does not seem to be supported by the "
+                          "current qemu version. Looking for a replacement...",
+                          flag)
+            supported_flag = self.get_qemu_flag_replacement(flag)
+            if supported_flag:
+                logging.debug("Replacing flag %s with %s", flag,
+                              supported_flag)
+                extra_params = extra_params.replace(flag, supported_flag)
+            else:
+                logging.error("No valid replacement was found for flag %s.",
+                              flag)
+
+        return extra_params
+
+
+    def get_qemu_flag_replacement(self, option):
+        """
+        Searches on a list of known misspellings for qemu options and returns
+        a replacement. If no replacement can be found, return None.
+
+        @param option: String representing qemu option (such as -mem).
+
+        @return: Option replacement, or None, if none found.
+        """
+        list_mispellings = [['-mem-path', '-mempath'],]
+        replacement = None
+
+        for mispellings in list_mispellings:
+            if option in mispellings:
+                option_position = mispellings.index(option)
+                replacement = mispellings[1 - option_position]
+
+        if replacement not in self.qemu_supported_flags:
+            logging.error("Replacement %s also does not seem to be a valid "
+                          "qemu flag, aborting replacement.", replacement)
+            return None
+
+        return replacement
-- 
1.6.2.5

Re: [KVM AUTOTEST PATCH] [RFC] KVM test: keep record of supported qemu options

[Lukáš Doktor]

Hello Lucas,

I like your patch but I'm not entirely convinced about it necessity. 
Stable version of KVM should have this fixed and unstable ones are for 
developers, who are skilled enough to fix this using kvm_test.cfg.

On the other hand keep this patch somewhere. Eventually if qemu started 
to be naughty, we would have something useful in our pocket.

Best regards,
Lukáš

Dne 29.7.2009 05:40, Lucas Meneghel Rodrigues napsal(a):
> In order to make it easier to figure out problems and
> also to avoid aborting tests prematurely due to
> incompatible qemu options, keep record of supported
> qemu options, and if extra options are passed to qemu,
> verify if they are amongst the supported options. Also,
> try to replace known misspelings on options in case
> something goes wrong, and be generous logging any problems.
>
> This first version of the patch gets supported flags from
> the output of qemu --help. I thought this would be good
> enough for a first start. I am asking for input on whether
> this is needed, and if yes, if the approach looks good.
>
> Signed-off-by: Lucas Meneghel Rodrigues<lmr@redhat.com>
> ---
>   client/tests/kvm/kvm_vm.py |   79 ++++++++++++++++++++++++++++++++++++++++++-
>   1 files changed, 77 insertions(+), 2 deletions(-)
>
> diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py
> index eba9b84..0dd34c2 100644
> --- a/client/tests/kvm/kvm_vm.py
> +++ b/client/tests/kvm/kvm_vm.py
> @@ -121,6 +121,7 @@ class VM:
>           self.qemu_path = qemu_path
>           self.image_dir = image_dir
>           self.iso_dir = iso_dir
> +        self.qemu_supported_flags = self.get_qemu_supported_flags()
>
>
>           # Find available monitor filename
> @@ -258,7 +259,7 @@ class VM:
>
>           extra_params = params.get("extra_params")
>           if extra_params:
> -            qemu_cmd += " %s" % extra_params
> +            qemu_cmd += " %s" % self.process_qemu_extra_params(extra_params)
>
>           for redir_name in kvm_utils.get_sub_dict_names(params, "redirs"):
>               redir_params = kvm_utils.get_sub_dict(params, redir_name)
> @@ -751,7 +752,7 @@ class VM:
>               else:
>                   self.send_key(char)
>
> -
> +
>       def get_uuid(self):
>           """
>           Catch UUID of the VM.
> @@ -762,3 +763,77 @@ class VM:
>               return self.uuid
>           else:
>               return self.params.get("uuid", None)
> +
> +
> +    def get_qemu_supported_flags(self):
> +        """
> +        Gets all supported qemu options from qemu-help. This is a useful
> +        procedure to quickly spot problems with incompatible qemu flags.
> +        """
> +        cmd = self.qemu_path + ' --help'
> +        (status, output) = kvm_subprocess.run_fg(cmd)
> +        supported_flags = []
> +
> +        if status:
> +            logging.error('Process qemu --help ended with exit code !=0. '
> +                          'No supported qemu flags will be recorded.')
> +            return supported_flags
> +
> +        for line in output.split('\n'):
> +            if line and line.startswith('-'):
> +                flag = line.split()[0]
> +                if flag not in supported_flags:
> +                    supported_flags.append(flag)
> +
> +        return supported_flags
> +
> +
> +    def process_qemu_extra_params(self, extra_params):
> +        """
> +        Verifies an extra param passed to qemu to see if it's supported by the
> +        current qemu version. If it's not supported, try to find an appropriate
> +        replacement on a list of known option misspellings.
> +
> +        @param extra_params: String with a qemu command line option.
> +        """
> +        flag = extra_params.split()[0]
> +
> +        if flag not in self.qemu_supported_flags:
> +            logging.error("Flag %s does not seem to be supported by the "
> +                          "current qemu version. Looking for a replacement...",
> +                          flag)
> +            supported_flag = self.get_qemu_flag_replacement(flag)
> +            if supported_flag:
> +                logging.debug("Replacing flag %s with %s", flag,
> +                              supported_flag)
> +                extra_params = extra_params.replace(flag, supported_flag)
> +            else:
> +                logging.error("No valid replacement was found for flag %s.",
> +                              flag)
> +
> +        return extra_params
> +
> +
> +    def get_qemu_flag_replacement(self, option):
> +        """
> +        Searches on a list of known misspellings for qemu options and returns
> +        a replacement. If no replacement can be found, return None.
> +
> +        @param option: String representing qemu option (such as -mem).
> +
> +        @return: Option replacement, or None, if none found.
> +        """
> +        list_mispellings = [['-mem-path', '-mempath'],]
> +        replacement = None
> +
> +        for mispellings in list_mispellings:
> +            if option in mispellings:
> +                option_position = mispellings.index(option)
> +                replacement = mispellings[1 - option_position]
> +
> +        if replacement not in self.qemu_supported_flags:
> +            logging.error("Replacement %s also does not seem to be a valid "
> +                          "qemu flag, aborting replacement.", replacement)
> +            return None
> +
> +        return replacement

Re: [Autotest] [KVM AUTOTEST PATCH] [RFC] KVM test: keep record of supported qemu options

[Lucas Meneghel Rodrigues]

On Wed, Jul 29, 2009 at 5:11 AM, Lukáš Doktor<ldoktor@redhat.com> wrote:
> Hello Lucas,
>
> I like your patch but I'm not entirely convinced about it necessity.
> Stable version of KVM should have this fixed and unstable ones are for
> developers, who are skilled enough to fix this using kvm_test.cfg.
>
> On the other hand keep this patch somewhere. Eventually if qemu started
> to be naughty, we would have something useful in our pocket.

Yep, I agree :) Thank you for your input!

> Best regards,
> Lukáš
>
> Dne 29.7.2009 05:40, Lucas Meneghel Rodrigues napsal(a):
>> In order to make it easier to figure out problems and
>> also to avoid aborting tests prematurely due to
>> incompatible qemu options, keep record of supported
>> qemu options, and if extra options are passed to qemu,
>> verify if they are amongst the supported options. Also,
>> try to replace known misspelings on options in case
>> something goes wrong, and be generous logging any problems.
>>
>> This first version of the patch gets supported flags from
>> the output of qemu --help. I thought this would be good
>> enough for a first start. I am asking for input on whether
>> this is needed, and if yes, if the approach looks good.
>>
>> Signed-off-by: Lucas Meneghel Rodrigues<lmr@redhat.com>
>> ---
>>   client/tests/kvm/kvm_vm.py |   79 ++++++++++++++++++++++++++++++++++++++++++-
>>   1 files changed, 77 insertions(+), 2 deletions(-)
>>
>> diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py
>> index eba9b84..0dd34c2 100644
>> --- a/client/tests/kvm/kvm_vm.py
>> +++ b/client/tests/kvm/kvm_vm.py
>> @@ -121,6 +121,7 @@ class VM:
>>           self.qemu_path = qemu_path
>>           self.image_dir = image_dir
>>           self.iso_dir = iso_dir
>> +        self.qemu_supported_flags = self.get_qemu_supported_flags()
>>
>>
>>           # Find available monitor filename
>> @@ -258,7 +259,7 @@ class VM:
>>
>>           extra_params = params.get("extra_params")
>>           if extra_params:
>> -            qemu_cmd += " %s" % extra_params
>> +            qemu_cmd += " %s" % self.process_qemu_extra_params(extra_params)
>>
>>           for redir_name in kvm_utils.get_sub_dict_names(params, "redirs"):
>>               redir_params = kvm_utils.get_sub_dict(params, redir_name)
>> @@ -751,7 +752,7 @@ class VM:
>>               else:
>>                   self.send_key(char)
>>
>> -
>> +
>>       def get_uuid(self):
>>           """
>>           Catch UUID of the VM.
>> @@ -762,3 +763,77 @@ class VM:
>>               return self.uuid
>>           else:
>>               return self.params.get("uuid", None)
>> +
>> +
>> +    def get_qemu_supported_flags(self):
>> +        """
>> +        Gets all supported qemu options from qemu-help. This is a useful
>> +        procedure to quickly spot problems with incompatible qemu flags.
>> +        """
>> +        cmd = self.qemu_path + ' --help'
>> +        (status, output) = kvm_subprocess.run_fg(cmd)
>> +        supported_flags = []
>> +
>> +        if status:
>> +            logging.error('Process qemu --help ended with exit code !=0. '
>> +                          'No supported qemu flags will be recorded.')
>> +            return supported_flags
>> +
>> +        for line in output.split('\n'):
>> +            if line and line.startswith('-'):
>> +                flag = line.split()[0]
>> +                if flag not in supported_flags:
>> +                    supported_flags.append(flag)
>> +
>> +        return supported_flags
>> +
>> +
>> +    def process_qemu_extra_params(self, extra_params):
>> +        """
>> +        Verifies an extra param passed to qemu to see if it's supported by the
>> +        current qemu version. If it's not supported, try to find an appropriate
>> +        replacement on a list of known option misspellings.
>> +
>> +        @param extra_params: String with a qemu command line option.
>> +        """
>> +        flag = extra_params.split()[0]
>> +
>> +        if flag not in self.qemu_supported_flags:
>> +            logging.error("Flag %s does not seem to be supported by the "
>> +                          "current qemu version. Looking for a replacement...",
>> +                          flag)
>> +            supported_flag = self.get_qemu_flag_replacement(flag)
>> +            if supported_flag:
>> +                logging.debug("Replacing flag %s with %s", flag,
>> +                              supported_flag)
>> +                extra_params = extra_params.replace(flag, supported_flag)
>> +            else:
>> +                logging.error("No valid replacement was found for flag %s.",
>> +                              flag)
>> +
>> +        return extra_params
>> +
>> +
>> +    def get_qemu_flag_replacement(self, option):
>> +        """
>> +        Searches on a list of known misspellings for qemu options and returns
>> +        a replacement. If no replacement can be found, return None.
>> +
>> +        @param option: String representing qemu option (such as -mem).
>> +
>> +        @return: Option replacement, or None, if none found.
>> +        """
>> +        list_mispellings = [['-mem-path', '-mempath'],]
>> +        replacement = None
>> +
>> +        for mispellings in list_mispellings:
>> +            if option in mispellings:
>> +                option_position = mispellings.index(option)
>> +                replacement = mispellings[1 - option_position]
>> +
>> +        if replacement not in self.qemu_supported_flags:
>> +            logging.error("Replacement %s also does not seem to be a valid "
>> +                          "qemu flag, aborting replacement.", replacement)
>> +            return None
>> +
>> +        return replacement
>
> _______________________________________________
> Autotest mailing list
> Autotest@test.kernel.org
> http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
>



-- 
Lucas Meneghel

Re: [KVM-AUTOTEST PATCH] KVM test: Add hugepage variant

[Lukáš Doktor]

Dne 29.7.2009 05:40, Lucas Meneghel Rodrigues napsal(a):
> This patch adds a small setup script to set up huge memory
> pages during the kvm tests execution. Also, added hugepage setup to the
> fc8_quick sample.
>
> Signed-off-by: Lukáš Doktor<ldoktor@redhat.com>
> Signed-off-by: Lucas Meneghel Rodrigues<lmr@redhat.com>
> ---
>   client/tests/kvm/kvm_tests.cfg.sample |    8 +++
>   client/tests/kvm/kvm_vm.py            |   11 +++
>   client/tests/kvm/scripts/hugepage.py  |  109 +++++++++++++++++++++++++++++++++
>   3 files changed, 128 insertions(+), 0 deletions(-)
>   create mode 100644 client/tests/kvm/scripts/hugepage.py
>
> diff --git a/client/tests/kvm/kvm_tests.cfg.sample b/client/tests/kvm/kvm_tests.cfg.sample
> index 2d75a66..7cd12cb 100644
> --- a/client/tests/kvm/kvm_tests.cfg.sample
> +++ b/client/tests/kvm/kvm_tests.cfg.sample
> @@ -587,6 +587,13 @@ variants:
>
>
>   variants:
> +    - @kvm_smallpages:
> +    - kvm_hugepages:
> +        pre_command = "/usr/bin/python scripts/hugepage.py /mnt/kvm_hugepage"
> +        extra_params += " -mem-path /mnt/kvm_hugepage"
> +
> +
> +variants:
>       - @basic:
>           only Fedora Windows
>       - @full:
> @@ -598,6 +605,7 @@ variants:
>           only Fedora.8.32
>           only install setup boot shutdown
>           only rtl8139
> +        only kvm_hugepages
>       - @sample1:
>           only qcow2
>           only ide
> diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py
> index d96b359..eba9b84 100644
> --- a/client/tests/kvm/kvm_vm.py
> +++ b/client/tests/kvm/kvm_vm.py
> @@ -397,6 +397,17 @@ class VM:
>                   self.destroy()
>                   return False
>
> +            # Get the output so far, to see if we have any problems with
> +            # hugepage setup.
> +            output = self.process.get_output()
> +
> +            if "alloc_mem_area" in output:
> +                logging.error("Could not allocate hugepage memory; "
> +                              "qemu command:\n%s" % qemu_command)
> +                logging.error("Output:" + kvm_utils.format_str_for_message(
> +                              self.process.get_output()))
> +                return False
> +
>               logging.debug("VM appears to be alive with PID %d",
>                             self.process.get_pid())
>               return True
> diff --git a/client/tests/kvm/scripts/hugepage.py b/client/tests/kvm/scripts/hugepage.py
> new file mode 100644
> index 0000000..dc36da4
> --- /dev/null
> +++ b/client/tests/kvm/scripts/hugepage.py
> @@ -0,0 +1,109 @@
> +#!/usr/bin/python
> +# -*- coding: utf-8 -*-
> +import os, sys, time
> +
> +"""
> +Simple script to allocate enough hugepages for KVM testing purposes.
> +"""
> +
> +class HugePageError(Exception):
> +    """
> +    Simple wrapper for the builtin Exception class.
> +    """
> +    pass
> +
> +
> +class HugePage:
> +    def __init__(self, hugepage_path=None):
> +        """
> +        Gets environment variable values and calculates the target number
> +        of huge memory pages.
> +
> +        @param hugepage_path: Path where to mount hugetlbfs path, if not
> +                yet configured.
> +        """
> +        self.vms = len(os.environ['KVM_TEST_vms'].split())
> +        self.mem = int(os.environ['KVM_TEST_mem'])
> +        try:
> +            self.max_vms = int(os.environ['KVM_TEST_max_vms'])
> +        except KeyError:
> +            self.max_vms = 0
> +
> +        if hugepage_path:
> +            self.hugepage_path = hugepage_path
> +        else:
> +            self.hugepage_path = '/mnt/kvm_hugepage'
> +
> +        self.hugepage_size = self.get_hugepage_size()
> +        self.target_hugepages = self.get_target_hugepages()
> +
> +
> +    def get_hugepage_size(self):
> +        """
> +        Get the current system setting for huge memory page size.
> +        """
> +        meminfo = open('/proc/meminfo', 'r').readlines()
> +        huge_line_list = [h for h in meminfo if h.startswith("Hugepagesize")]
> +        try:
> +            return int(huge_line_list[0].split()[1])
> +        except ValueError, e:
> +            raise HugePageError("Could not get huge page size setting from "
> +                                "/proc/meminfo: %s" % e)
> +
> +
> +    def get_target_hugepages(self):
> +        """
> +        Calculate the target number of hugepages for testing purposes.
> +        """
> +        if self.vms<  self.max_vms:
> +            self.vms = self.max_vms
> +        vmsm = (self.vms * self.mem) + (self.vms * 64)
> +        return int(vmsm * 1024 / self.hugepage_size)
> +
> +
> +    def set_hugepages(self):
> +        """
> +        Sets the hugepage limit to the target hugepage value calculated.
> +        """
> +        hugepage_cfg = open("/proc/sys/vm/nr_hugepages", "r+")
> +        hp = hugepage_cfg.readline()
> +        while int(hp)<  self.target_hugepages:
> +            loop_hp = hp
> +            hugepage_cfg.write(str(self.target_hugepages))
> +            hugepage_cfg.flush()
> +            hugepage_cfg.seek(0)
> +            hp = int(hugepage_cfg.readline())
> +            if loop_hp == hp:
> +                raise HugePageError("Cannot set the kernel hugepage setting "
> +                                    "to the target value of %d hugepages." %
> +                                    self.target_hugepages)
> +        hugepage_cfg.close()
> +
> +
> +    def mount_hugepage_fs(self):
> +        """
> +        Verify if there's a hugetlbfs mount set. If there's none, will set up
> +        a hugetlbfs mount using the class attribute that defines the mount
> +        point.
> +        """
> +        if not os.path.ismount(self.hugepage_path):
> +            if not os.path.isdir(self.hugepage_path):
> +                os.makedirs(self.hugepage_path)
> +            cmd = "mount -t hugetlbfs none %s" % self.hugepage_path
> +            if os.system(cmd):
> +                raise HugePageError("Cannot mount hugetlbfs path %s" %
> +                                    self.hugepage_path)
> +
> +
> +    def setup(self):
> +        self.set_hugepages()
> +        self.mount_hugepage_fs()
> +
> +
> +if __name__ == "__main__":
> +    if len(sys.argv)<  2:
> +        huge_page = HugePage()
> +    else:
> +        huge_page = HugePage(sys.argv[1])
> +
> +    huge_page.setup()


Acked-by: Lukáš Doktor <ldoktor@redhat.com>

Re: [KVM-AUTOTEST PATCH] KVM test: Add hugepage variant

[Ryan Harper]

* Lucas Meneghel Rodrigues <lmr@redhat.com> [2009-07-28 22:40]:
> This patch adds a small setup script to set up huge memory
> pages during the kvm tests execution. Also, added hugepage setup to the
> fc8_quick sample.
> 
> Signed-off-by: Luká?? Doktor <ldoktor@redhat.com>
> Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>

Looks good.  one nit below.

Signed-off-by: Ryan Harper <ryanh@us.ibm.com>

> ---
>  client/tests/kvm/kvm_tests.cfg.sample |    8 +++
>  client/tests/kvm/kvm_vm.py            |   11 +++
>  client/tests/kvm/scripts/hugepage.py  |  109 +++++++++++++++++++++++++++++++++
>  3 files changed, 128 insertions(+), 0 deletions(-)
>  create mode 100644 client/tests/kvm/scripts/hugepage.py
> 
> diff --git a/client/tests/kvm/kvm_tests.cfg.sample b/client/tests/kvm/kvm_tests.cfg.sample
> index 2d75a66..7cd12cb 100644
> --- a/client/tests/kvm/kvm_tests.cfg.sample
> +++ b/client/tests/kvm/kvm_tests.cfg.sample
> @@ -587,6 +587,13 @@ variants:
> 
> 
>  variants:
> +    - @kvm_smallpages:
> +    - kvm_hugepages:
> +        pre_command = "/usr/bin/python scripts/hugepage.py /mnt/kvm_hugepage"
> +        extra_params += " -mem-path /mnt/kvm_hugepage"
> +
> +
> +variants:
>      - @basic:
>          only Fedora Windows
>      - @full:
> @@ -598,6 +605,7 @@ variants:
>          only Fedora.8.32
>          only install setup boot shutdown
>          only rtl8139
> +        only kvm_hugepages
>      - @sample1:
>          only qcow2
>          only ide
> diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py
> index d96b359..eba9b84 100644
> --- a/client/tests/kvm/kvm_vm.py
> +++ b/client/tests/kvm/kvm_vm.py
> @@ -397,6 +397,17 @@ class VM:
>                  self.destroy()
>                  return False
> 
> +            # Get the output so far, to see if we have any problems with
> +            # hugepage setup.
> +            output = self.process.get_output()
> +
> +            if "alloc_mem_area" in output:
> +                logging.error("Could not allocate hugepage memory; "
> +                              "qemu command:\n%s" % qemu_command)
> +                logging.error("Output:" + kvm_utils.format_str_for_message(
> +                              self.process.get_output()))
> +                return False
> +
>              logging.debug("VM appears to be alive with PID %d",
>                            self.process.get_pid())
>              return True
> diff --git a/client/tests/kvm/scripts/hugepage.py b/client/tests/kvm/scripts/hugepage.py
> new file mode 100644
> index 0000000..dc36da4
> --- /dev/null
> +++ b/client/tests/kvm/scripts/hugepage.py
> @@ -0,0 +1,109 @@
> +#!/usr/bin/python
> +# -*- coding: utf-8 -*-
> +import os, sys, time
> +
> +"""
> +Simple script to allocate enough hugepages for KVM testing purposes.
> +"""
> +
> +class HugePageError(Exception):
> +    """
> +    Simple wrapper for the builtin Exception class.
> +    """
> +    pass
> +
> +
> +class HugePage:
> +    def __init__(self, hugepage_path=None):
> +        """
> +        Gets environment variable values and calculates the target number
> +        of huge memory pages.
> +
> +        @param hugepage_path: Path where to mount hugetlbfs path, if not
> +                yet configured.
> +        """
> +        self.vms = len(os.environ['KVM_TEST_vms'].split())
> +        self.mem = int(os.environ['KVM_TEST_mem'])
> +        try:
> +            self.max_vms = int(os.environ['KVM_TEST_max_vms'])
> +        except KeyError:
> +            self.max_vms = 0
> +
> +        if hugepage_path:
> +            self.hugepage_path = hugepage_path
> +        else:
> +            self.hugepage_path = '/mnt/kvm_hugepage'
> +
> +        self.hugepage_size = self.get_hugepage_size()
> +        self.target_hugepages = self.get_target_hugepages()
> +
> +
> +    def get_hugepage_size(self):
> +        """
> +        Get the current system setting for huge memory page size.
> +        """
> +        meminfo = open('/proc/meminfo', 'r').readlines()
> +        huge_line_list = [h for h in meminfo if h.startswith("Hugepagesize")]
> +        try:
> +            return int(huge_line_list[0].split()[1])
> +        except ValueError, e:
> +            raise HugePageError("Could not get huge page size setting from "
> +                                "/proc/meminfo: %s" % e)
> +
> +
> +    def get_target_hugepages(self):
> +        """
> +        Calculate the target number of hugepages for testing purposes.
> +        """
> +        if self.vms < self.max_vms:
> +            self.vms = self.max_vms
> +        vmsm = (self.vms * self.mem) + (self.vms * 64)

Nit: Maybe a comment about the fudge factor being added in?

> +        return int(vmsm * 1024 / self.hugepage_size)
> +
> +
> +    def set_hugepages(self):
> +        """
> +        Sets the hugepage limit to the target hugepage value calculated.
> +        """
> +        hugepage_cfg = open("/proc/sys/vm/nr_hugepages", "r+")
> +        hp = hugepage_cfg.readline()
> +        while int(hp) < self.target_hugepages:
> +            loop_hp = hp
> +            hugepage_cfg.write(str(self.target_hugepages))
> +            hugepage_cfg.flush()
> +            hugepage_cfg.seek(0)
> +            hp = int(hugepage_cfg.readline())
> +            if loop_hp == hp:
> +                raise HugePageError("Cannot set the kernel hugepage setting "
> +                                    "to the target value of %d hugepages." %
> +                                    self.target_hugepages)
> +        hugepage_cfg.close()
> +
> +
> +    def mount_hugepage_fs(self):
> +        """
> +        Verify if there's a hugetlbfs mount set. If there's none, will set up
> +        a hugetlbfs mount using the class attribute that defines the mount
> +        point.
> +        """
> +        if not os.path.ismount(self.hugepage_path):
> +            if not os.path.isdir(self.hugepage_path):
> +                os.makedirs(self.hugepage_path)
> +            cmd = "mount -t hugetlbfs none %s" % self.hugepage_path
> +            if os.system(cmd):
> +                raise HugePageError("Cannot mount hugetlbfs path %s" %
> +                                    self.hugepage_path)
> +
> +
> +    def setup(self):
> +        self.set_hugepages()
> +        self.mount_hugepage_fs()
> +
> +
> +if __name__ == "__main__":
> +    if len(sys.argv) < 2:
> +        huge_page = HugePage()
> +    else:
> +        huge_page = HugePage(sys.argv[1])
> +
> +    huge_page.setup()
> -- 
> 1.6.2.5
> 
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

-- 
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
ryanh@us.ibm.com

Re: [KVM-AUTOTEST PATCH] KVM test: Add hugepage variant

[Lukáš Doktor]

Hello Ryan,

see below...

Dne 29.7.2009 16:41, Ryan Harper napsal(a):
> * Lucas Meneghel Rodrigues<lmr@redhat.com>  [2009-07-28 22:40]:
>> This patch adds a small setup script to set up huge memory
>> pages during the kvm tests execution. Also, added hugepage setup to the
>> fc8_quick sample.
>>
>> Signed-off-by: Luká?? Doktor<ldoktor@redhat.com>
>> Signed-off-by: Lucas Meneghel Rodrigues<lmr@redhat.com>
>
> Looks good.  one nit below.
>
> Signed-off-by: Ryan Harper<ryanh@us.ibm.com>
>
>> ---
>> +
>> +    def get_target_hugepages(self):
>> +        """
>> +        Calculate the target number of hugepages for testing purposes.
>> +        """
>> +        if self.vms<  self.max_vms:
>> +            self.vms = self.max_vms
>> +        vmsm = (self.vms * self.mem) + (self.vms * 64)
>
> Nit: Maybe a comment about the fudge factor being added in?

It's qemu-kvm overhead. Should I change the patch or is this explanation 
sufficient?

Thanks for the feedback,
bye, Lukáš

Re: [KVM AUTOTEST PATCH] [RFC] KVM test: keep record of supported qemu options

[Michael Goldish]

I'm not entirely sure this is needed:
- How often do tests fail due to unsupported flags? I thought the -mem-path
problem was only due to a temporary typo (I'm not sure how temporary).
Once it's fixed, will we ever encounter that misspelling again?
- A typo is a bug too, so maybe we shouldn't make an effort to support it.
- Maybe it's not so bad to ask users to change the config file when they
want the system to work with a known typo. That's why we put stuff in the
config file rather than hardcode it into Autotest -- the config file can
be modified more easily.

Also, it _might_ not work:
- "-help" is sometimes outdated as far as I know. Right now it seems up to
date, but not long ago it didn't mention -incoming and I think -rtc-td-hack
was missing too, so I'm not sure relying on it is safe.
- If -mem-path is a typo, what appears in the output of -help? The typo or
the correct spelling? If the correct spelling appears, then -help is of no
use to us.

Otherwise, the implementation looks nice. I'm not sure I'm right about the
things listed above so please tell me what you think.

Thanks,
Michael

----- Original Message -----
From: "Lucas Meneghel Rodrigues" <lmr@redhat.com>
To: autotest@test.kernel.org
Cc: kvm@vger.kernel.org, ldoktor@redhat.com, ryanh@us.ibm.com, mgoldish@redhat.com, "Lucas Meneghel Rodrigues" <lmr@redhat.com>
Sent: Wednesday, July 29, 2009 6:40:29 AM (GMT+0200) Auto-Detected
Subject: [KVM AUTOTEST PATCH] [RFC] KVM test: keep record of supported qemu options

In order to make it easier to figure out problems and
also to avoid aborting tests prematurely due to
incompatible qemu options, keep record of supported
qemu options, and if extra options are passed to qemu,
verify if they are amongst the supported options. Also,
try to replace known misspelings on options in case
something goes wrong, and be generous logging any problems.

This first version of the patch gets supported flags from
the output of qemu --help. I thought this would be good
enough for a first start. I am asking for input on whether
this is needed, and if yes, if the approach looks good.

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/kvm_vm.py |   79 ++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 77 insertions(+), 2 deletions(-)

diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py
index eba9b84..0dd34c2 100644
--- a/client/tests/kvm/kvm_vm.py
+++ b/client/tests/kvm/kvm_vm.py
@@ -121,6 +121,7 @@ class VM:
         self.qemu_path = qemu_path
         self.image_dir = image_dir
         self.iso_dir = iso_dir
+        self.qemu_supported_flags = self.get_qemu_supported_flags()
 
 
         # Find available monitor filename
@@ -258,7 +259,7 @@ class VM:
 
         extra_params = params.get("extra_params")
         if extra_params:
-            qemu_cmd += " %s" % extra_params
+            qemu_cmd += " %s" % self.process_qemu_extra_params(extra_params)
 
         for redir_name in kvm_utils.get_sub_dict_names(params, "redirs"):
             redir_params = kvm_utils.get_sub_dict(params, redir_name)
@@ -751,7 +752,7 @@ class VM:
             else:
                 self.send_key(char)
 
-    
+
     def get_uuid(self):
         """
         Catch UUID of the VM.
@@ -762,3 +763,77 @@ class VM:
             return self.uuid
         else:
             return self.params.get("uuid", None)
+
+
+    def get_qemu_supported_flags(self):
+        """
+        Gets all supported qemu options from qemu-help. This is a useful
+        procedure to quickly spot problems with incompatible qemu flags.
+        """
+        cmd = self.qemu_path + ' --help'
+        (status, output) = kvm_subprocess.run_fg(cmd)
+        supported_flags = []
+
+        if status:
+            logging.error('Process qemu --help ended with exit code !=0. '
+                          'No supported qemu flags will be recorded.')
+            return supported_flags
+
+        for line in output.split('\n'):
+            if line and line.startswith('-'):
+                flag = line.split()[0]
+                if flag not in supported_flags:
+                    supported_flags.append(flag)
+
+        return supported_flags
+
+
+    def process_qemu_extra_params(self, extra_params):
+        """
+        Verifies an extra param passed to qemu to see if it's supported by the
+        current qemu version. If it's not supported, try to find an appropriate
+        replacement on a list of known option misspellings.
+
+        @param extra_params: String with a qemu command line option.
+        """
+        flag = extra_params.split()[0]
+
+        if flag not in self.qemu_supported_flags:
+            logging.error("Flag %s does not seem to be supported by the "
+                          "current qemu version. Looking for a replacement...",
+                          flag)
+            supported_flag = self.get_qemu_flag_replacement(flag)
+            if supported_flag:
+                logging.debug("Replacing flag %s with %s", flag,
+                              supported_flag)
+                extra_params = extra_params.replace(flag, supported_flag)
+            else:
+                logging.error("No valid replacement was found for flag %s.",
+                              flag)
+
+        return extra_params
+
+
+    def get_qemu_flag_replacement(self, option):
+        """
+        Searches on a list of known misspellings for qemu options and returns
+        a replacement. If no replacement can be found, return None.
+
+        @param option: String representing qemu option (such as -mem).
+
+        @return: Option replacement, or None, if none found.
+        """
+        list_mispellings = [['-mem-path', '-mempath'],]
+        replacement = None
+
+        for mispellings in list_mispellings:
+            if option in mispellings:
+                option_position = mispellings.index(option)
+                replacement = mispellings[1 - option_position]
+
+        if replacement not in self.qemu_supported_flags:
+            logging.error("Replacement %s also does not seem to be a valid "
+                          "qemu flag, aborting replacement.", replacement)
+            return None
+
+        return replacement
-- 
1.6.2.5