[PATCH 0/2] Removing scripts/hugepage.py

[PATCH 0/2] Removing scripts/hugepage.py

[Lucas Meneghel Rodrigues]

Continuing on our quest of obsoleting and removing the pre/post
scripts, this time we remove hugepage.py for a slightly better
and cleaner framework implementation.

Lucas Meneghel Rodrigues (2):
  KVM test: Turning hugepages setup into KVM autotest infrastructure
  KVM test: Removing scripts/hugepage.py

 client/tests/kvm/kvm_preprocessing.py  |    8 ++
 client/tests/kvm/kvm_utils.py          |  102 +++++++++++++++++++++++++++
 client/tests/kvm/scripts/hugepage.py   |  118 --------------------------------
 client/tests/kvm/tests_base.cfg.sample |    3 +-
 4 files changed, 111 insertions(+), 120 deletions(-)
 delete mode 100755 client/tests/kvm/scripts/hugepage.py

-- 
1.7.3.4

[PATCH 1/2] KVM test: Turning hugepages setup into KVM autotest infrastructure

[Lucas Meneghel Rodrigues]

So we can get rid of scripts/hugepage.py. The implementation
strategy is to have a kvm_utils.HugePageConfig class that
can do both hugepages setup and cleanup, and call it during
pre/postprocessing.

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/kvm_preprocessing.py  |    8 +++
 client/tests/kvm/kvm_utils.py          |  102 ++++++++++++++++++++++++++++++++
 client/tests/kvm/tests_base.cfg.sample |    3 +-
 3 files changed, 111 insertions(+), 2 deletions(-)

diff --git a/client/tests/kvm/kvm_preprocessing.py b/client/tests/kvm/kvm_preprocessing.py
index 4a6e0f8..41455cf 100644
--- a/client/tests/kvm/kvm_preprocessing.py
+++ b/client/tests/kvm/kvm_preprocessing.py
@@ -254,6 +254,10 @@ def preprocess(test, params, env):
     logging.debug("KVM userspace version: %s" % kvm_userspace_version)
     test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version})
 
+    if params.get("setup_hugepages") == "yes":
+        h = kvm_utils.HugePageConfig(params)
+        h.setup()
+
     # Execute any pre_commands
     if params.get("pre_command"):
         process_command(test, params, env, params.get("pre_command"),
@@ -350,6 +354,10 @@ def postprocess(test, params, env):
         env["tcpdump"].close()
         del env["tcpdump"]
 
+    if params.get("setup_hugepages") == "yes":
+        h = kvm_utils.HugePageConfig(params)
+        h.cleanup()
+
     # Execute any post_commands
     if params.get("post_command"):
         process_command(test, params, env, params.get("post_command"),
diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_utils.py
index 78c9f25..b61ff94 100644
--- a/client/tests/kvm/kvm_utils.py
+++ b/client/tests/kvm/kvm_utils.py
@@ -1265,6 +1265,108 @@ class KvmLoggingConfig(logging_config.LoggingConfig):
                                                         verbose=verbose)
 
 
+class HugePageConfig:
+    def __init__(self, params):
+        """
+        Gets environment variable values and calculates the target number
+        of huge memory pages.
+
+        @param params: Dict like object containing parameters for the test.
+        """
+        self.vms = len(params.objects("vms"))
+        self.mem = int(params.get("mem"))
+        self.max_vms = int(params.get("max_vms", 0))
+        self.hugepage_path = '/mnt/kvm_hugepage'
+        self.hugepage_size = self.get_hugepage_size()
+        self.target_hugepages = self.get_target_hugepages()
+        self.kernel_hp_file = '/proc/sys/vm/nr_hugepages'
+
+
+    def get_hugepage_size(self):
+        """
+        Get the current system setting for huge memory page size.
+        """
+        meminfo = open('/proc/meminfo', 'r').readlines()
+        huge_line_list = [h for h in meminfo if h.startswith("Hugepagesize")]
+        try:
+            return int(huge_line_list[0].split()[1])
+        except ValueError, e:
+            raise ValueError("Could not get huge page size setting from "
+                             "/proc/meminfo: %s" % e)
+
+
+    def get_target_hugepages(self):
+        """
+        Calculate the target number of hugepages for testing purposes.
+        """
+        if self.vms < self.max_vms:
+            self.vms = self.max_vms
+        # memory of all VMs plus qemu overhead of 64MB per guest
+        vmsm = (self.vms * self.mem) + (self.vms * 64)
+        return int(vmsm * 1024 / self.hugepage_size)
+
+
+    @error.context_aware
+    def set_hugepages(self):
+        """
+        Sets the hugepage limit to the target hugepage value calculated.
+        """
+        error.base_context("Setting hugepages limit to %s" %
+                           self.target_hugepages)
+        hugepage_cfg = open(self.kernel_hp_file, "r+")
+        hp = hugepage_cfg.readline()
+        while int(hp) < self.target_hugepages:
+            loop_hp = hp
+            hugepage_cfg.write(str(self.target_hugepages))
+            hugepage_cfg.flush()
+            hugepage_cfg.seek(0)
+            hp = int(hugepage_cfg.readline())
+            if loop_hp == hp:
+                raise ValueError("Cannot set the kernel hugepage setting "
+                                 "to the target value of %d hugepages." %
+                                 self.target_hugepages)
+        hugepage_cfg.close()
+        logging.debug("Successfuly set %s large memory pages on host ",
+                      self.target_hugepages)
+
+
+    @error.context_aware
+    def mount_hugepage_fs(self):
+        """
+        Verify if there's a hugetlbfs mount set. If there's none, will set up
+        a hugetlbfs mount using the class attribute that defines the mount
+        point.
+        """
+        error.base_context("Mounting hugepages path")
+        if not os.path.ismount(self.hugepage_path):
+            if not os.path.isdir(self.hugepage_path):
+                os.makedirs(self.hugepage_path)
+            cmd = "mount -t hugetlbfs none %s" % self.hugepage_path
+            utils.system(cmd)
+
+
+    def setup(self):
+        logging.debug("Number of VMs this test will use: %d", self.vms)
+        logging.debug("Amount of memory used by each vm: %s", self.mem)
+        logging.debug("System setting for large memory page size: %s",
+                      self.hugepage_size)
+        logging.debug("Number of large memory pages needed for this test: %s",
+                      self.target_hugepages)
+        self.set_hugepages()
+        self.mount_hugepage_fs()
+
+
+    @error.context_aware
+    def cleanup(self):
+        error.base_context("Trying to dealocate hugepage memory")
+        try:
+            utils.system("umount %s" % self.hugepage_path)
+        except error.CmdError:
+            return
+        utils.system("echo 0 > %s" % self.kernel_hp_file)
+        logging.debug("Hugepage memory successfuly dealocated")
+
+
 class PciAssignable(object):
     """
     Request PCI assignable devices on host. It will check whether to request
diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample
index 8b67256..b45d821 100644
--- a/client/tests/kvm/tests_base.cfg.sample
+++ b/client/tests/kvm/tests_base.cfg.sample
@@ -2378,8 +2378,7 @@ variants:
 variants:
     - @smallpages:
     - hugepages:
-        pre_command += " scripts/hugepage.py /mnt/kvm_hugepage;"
-        post_command += " umount /mnt/kvm_hugepage && echo 0 > /proc/sys/vm/nr_hugepages;"
+        setup_hugepages = yes
         extra_params += " -mem-path /mnt/kvm_hugepage"
 
 
-- 
1.7.3.4

[PATCH 2/2] KVM test: Removing scripts/hugepage.py

[Lucas Meneghel Rodrigues]

Now that its functionality has been reimplemented as
KVM test infrastructure.

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/scripts/hugepage.py |  118 ----------------------------------
 1 files changed, 0 insertions(+), 118 deletions(-)
 delete mode 100755 client/tests/kvm/scripts/hugepage.py

diff --git a/client/tests/kvm/scripts/hugepage.py b/client/tests/kvm/scripts/hugepage.py
deleted file mode 100755
index 8a1b0f6..0000000
--- a/client/tests/kvm/scripts/hugepage.py
+++ /dev/null
@@ -1,118 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-import os, sys, time
-
-"""
-Simple script to allocate enough hugepages for KVM testing purposes.
-"""
-
-class HugePageError(Exception):
-    """
-    Simple wrapper for the builtin Exception class.
-    """
-    pass
-
-
-class HugePage:
-    def __init__(self, hugepage_path=None):
-        """
-        Gets environment variable values and calculates the target number
-        of huge memory pages.
-
-        @param hugepage_path: Path where to mount hugetlbfs path, if not
-                yet configured.
-        """
-        self.vms = len(os.environ['KVM_TEST_vms'].split())
-        self.mem = int(os.environ['KVM_TEST_mem'])
-        try:
-            self.max_vms = int(os.environ['KVM_TEST_max_vms'])
-        except KeyError:
-            self.max_vms = 0
-
-        if hugepage_path:
-            self.hugepage_path = hugepage_path
-        else:
-            self.hugepage_path = '/mnt/kvm_hugepage'
-
-        self.hugepage_size = self.get_hugepage_size()
-        self.target_hugepages = self.get_target_hugepages()
-        print "Number of VMs this test will use: %d" % self.vms
-        print "Amount of memory used by each vm: %s" % self.mem
-        print ("System setting for large memory page size: %s" %
-               self.hugepage_size)
-        print ("Number of large memory pages needed for this test: %s" %
-               self.target_hugepages)
-
-
-    def get_hugepage_size(self):
-        """
-        Get the current system setting for huge memory page size.
-        """
-        meminfo = open('/proc/meminfo', 'r').readlines()
-        huge_line_list = [h for h in meminfo if h.startswith("Hugepagesize")]
-        try:
-            return int(huge_line_list[0].split()[1])
-        except ValueError, e:
-            raise HugePageError("Could not get huge page size setting from "
-                                "/proc/meminfo: %s" % e)
-
-
-    def get_target_hugepages(self):
-        """
-        Calculate the target number of hugepages for testing purposes.
-        """
-        if self.vms < self.max_vms:
-            self.vms = self.max_vms
-        # memory of all VMs plus qemu overhead of 64MB per guest
-        vmsm = (self.vms * self.mem) + (self.vms * 64)
-        return int(vmsm * 1024 / self.hugepage_size)
-
-
-    def set_hugepages(self):
-        """
-        Sets the hugepage limit to the target hugepage value calculated.
-        """
-        hugepage_cfg = open("/proc/sys/vm/nr_hugepages", "r+")
-        hp = hugepage_cfg.readline()
-        while int(hp) < self.target_hugepages:
-            loop_hp = hp
-            hugepage_cfg.write(str(self.target_hugepages))
-            hugepage_cfg.flush()
-            hugepage_cfg.seek(0)
-            hp = int(hugepage_cfg.readline())
-            if loop_hp == hp:
-                raise HugePageError("Cannot set the kernel hugepage setting "
-                                    "to the target value of %d hugepages." %
-                                    self.target_hugepages)
-        hugepage_cfg.close()
-        print ("Successfuly set %s large memory pages on host " %
-               self.target_hugepages)
-
-
-    def mount_hugepage_fs(self):
-        """
-        Verify if there's a hugetlbfs mount set. If there's none, will set up
-        a hugetlbfs mount using the class attribute that defines the mount
-        point.
-        """
-        if not os.path.ismount(self.hugepage_path):
-            if not os.path.isdir(self.hugepage_path):
-                os.makedirs(self.hugepage_path)
-            cmd = "mount -t hugetlbfs none %s" % self.hugepage_path
-            if os.system(cmd):
-                raise HugePageError("Cannot mount hugetlbfs path %s" %
-                                    self.hugepage_path)
-
-
-    def setup(self):
-        self.set_hugepages()
-        self.mount_hugepage_fs()
-
-
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        huge_page = HugePage()
-    else:
-        huge_page = HugePage(sys.argv[1])
-
-    huge_page.setup()
-- 
1.7.3.4

Re: [PATCH 1/2] KVM test: Turning hugepages setup into KVM autotest infrastructure

[Michael Goldish]

On 01/19/2011 03:56 AM, Lucas Meneghel Rodrigues wrote:
> So we can get rid of scripts/hugepage.py. The implementation
> strategy is to have a kvm_utils.HugePageConfig class that
> can do both hugepages setup and cleanup, and call it during
> pre/postprocessing.
> 
> Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
> ---
>  client/tests/kvm/kvm_preprocessing.py  |    8 +++
>  client/tests/kvm/kvm_utils.py          |  102 ++++++++++++++++++++++++++++++++
>  client/tests/kvm/tests_base.cfg.sample |    3 +-
>  3 files changed, 111 insertions(+), 2 deletions(-)
> 
> diff --git a/client/tests/kvm/kvm_preprocessing.py b/client/tests/kvm/kvm_preprocessing.py
> index 4a6e0f8..41455cf 100644
> --- a/client/tests/kvm/kvm_preprocessing.py
> +++ b/client/tests/kvm/kvm_preprocessing.py
> @@ -254,6 +254,10 @@ def preprocess(test, params, env):
>      logging.debug("KVM userspace version: %s" % kvm_userspace_version)
>      test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version})
>  
> +    if params.get("setup_hugepages") == "yes":
> +        h = kvm_utils.HugePageConfig(params)
> +        h.setup()
> +
>      # Execute any pre_commands
>      if params.get("pre_command"):
>          process_command(test, params, env, params.get("pre_command"),
> @@ -350,6 +354,10 @@ def postprocess(test, params, env):
>          env["tcpdump"].close()
>          del env["tcpdump"]
>  
> +    if params.get("setup_hugepages") == "yes":
> +        h = kvm_utils.HugePageConfig(params)
> +        h.cleanup()
> +
>      # Execute any post_commands
>      if params.get("post_command"):
>          process_command(test, params, env, params.get("post_command"),
> diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_utils.py
> index 78c9f25..b61ff94 100644
> --- a/client/tests/kvm/kvm_utils.py
> +++ b/client/tests/kvm/kvm_utils.py
> @@ -1265,6 +1265,108 @@ class KvmLoggingConfig(logging_config.LoggingConfig):
>                                                          verbose=verbose)
>  
>  
> +class HugePageConfig:
> +    def __init__(self, params):
> +        """
> +        Gets environment variable values and calculates the target number
> +        of huge memory pages.
> +
> +        @param params: Dict like object containing parameters for the test.
> +        """
> +        self.vms = len(params.objects("vms"))
> +        self.mem = int(params.get("mem"))
> +        self.max_vms = int(params.get("max_vms", 0))
> +        self.hugepage_path = '/mnt/kvm_hugepage'
> +        self.hugepage_size = self.get_hugepage_size()
> +        self.target_hugepages = self.get_target_hugepages()
> +        self.kernel_hp_file = '/proc/sys/vm/nr_hugepages'
> +
> +
> +    def get_hugepage_size(self):
> +        """
> +        Get the current system setting for huge memory page size.
> +        """
> +        meminfo = open('/proc/meminfo', 'r').readlines()
> +        huge_line_list = [h for h in meminfo if h.startswith("Hugepagesize")]
> +        try:
> +            return int(huge_line_list[0].split()[1])
> +        except ValueError, e:
> +            raise ValueError("Could not get huge page size setting from "
> +                             "/proc/meminfo: %s" % e)
> +
> +
> +    def get_target_hugepages(self):
> +        """
> +        Calculate the target number of hugepages for testing purposes.
> +        """
> +        if self.vms < self.max_vms:
> +            self.vms = self.max_vms
> +        # memory of all VMs plus qemu overhead of 64MB per guest
> +        vmsm = (self.vms * self.mem) + (self.vms * 64)
> +        return int(vmsm * 1024 / self.hugepage_size)
> +
> +
> +    @error.context_aware
> +    def set_hugepages(self):
> +        """
> +        Sets the hugepage limit to the target hugepage value calculated.
> +        """
> +        error.base_context("Setting hugepages limit to %s" %
> +                           self.target_hugepages)
> +        hugepage_cfg = open(self.kernel_hp_file, "r+")
> +        hp = hugepage_cfg.readline()
> +        while int(hp) < self.target_hugepages:
> +            loop_hp = hp
> +            hugepage_cfg.write(str(self.target_hugepages))
> +            hugepage_cfg.flush()
> +            hugepage_cfg.seek(0)
> +            hp = int(hugepage_cfg.readline())
> +            if loop_hp == hp:
> +                raise ValueError("Cannot set the kernel hugepage setting "
> +                                 "to the target value of %d hugepages." %
> +                                 self.target_hugepages)
> +        hugepage_cfg.close()
> +        logging.debug("Successfuly set %s large memory pages on host ",
> +                      self.target_hugepages)
> +
> +
> +    @error.context_aware
> +    def mount_hugepage_fs(self):
> +        """
> +        Verify if there's a hugetlbfs mount set. If there's none, will set up
> +        a hugetlbfs mount using the class attribute that defines the mount
> +        point.
> +        """
> +        error.base_context("Mounting hugepages path")
> +        if not os.path.ismount(self.hugepage_path):
> +            if not os.path.isdir(self.hugepage_path):
> +                os.makedirs(self.hugepage_path)
> +            cmd = "mount -t hugetlbfs none %s" % self.hugepage_path
> +            utils.system(cmd)
> +
> +
> +    def setup(self):
> +        logging.debug("Number of VMs this test will use: %d", self.vms)
> +        logging.debug("Amount of memory used by each vm: %s", self.mem)
> +        logging.debug("System setting for large memory page size: %s",
> +                      self.hugepage_size)
> +        logging.debug("Number of large memory pages needed for this test: %s",
> +                      self.target_hugepages)
> +        self.set_hugepages()
> +        self.mount_hugepage_fs()
> +
> +
> +    @error.context_aware
> +    def cleanup(self):
> +        error.base_context("Trying to dealocate hugepage memory")

Since there's just 1 context level in these functions, context() and
base_context() do the same thing, and context() is shorter.

Style note: in my uses of context() I didn't capitalize the first letter
of the string because it's always followed by 'context:' or by an arrow
in the final output.  If you think it's nicer to capitalize those
strings, we should be consistent and do it to all of them.

> +        try:
> +            utils.system("umount %s" % self.hugepage_path)
> +        except error.CmdError:
> +            return
> +        utils.system("echo 0 > %s" % self.kernel_hp_file)
> +        logging.debug("Hugepage memory successfuly dealocated")
> +
> +
>  class PciAssignable(object):
>      """
>      Request PCI assignable devices on host. It will check whether to request
> diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample
> index 8b67256..b45d821 100644
> --- a/client/tests/kvm/tests_base.cfg.sample
> +++ b/client/tests/kvm/tests_base.cfg.sample
> @@ -2378,8 +2378,7 @@ variants:
>  variants:
>      - @smallpages:
>      - hugepages:
> -        pre_command += " scripts/hugepage.py /mnt/kvm_hugepage;"
> -        post_command += " umount /mnt/kvm_hugepage && echo 0 > /proc/sys/vm/nr_hugepages;"
> +        setup_hugepages = yes
>          extra_params += " -mem-path /mnt/kvm_hugepage"
>  
>