* [PATCH 1/8] KVM test: Refactor the KVM build code into installer
2010-12-29 22:37 [PATCH 0/8] KVM test: Build code refactor + module_probe test Lucas Meneghel Rodrigues
@ 2010-12-29 22:37 ` Lucas Meneghel Rodrigues
2010-12-29 22:37 ` [PATCH 2/8] KVM test: rename 'mode' to 'install_mode' on the sample cfg files Lucas Meneghel Rodrigues
` (6 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Lucas Meneghel Rodrigues @ 2010-12-29 22:37 UTC (permalink / raw)
To: autotest; +Cc: Eduardo Habkost, kvm
In order to use the build infrastructure for other
tests, such the module probe test, refactor the build
code in a library, installer. This new library has
the base build code, with a number of improvements:
- Made code shorter and easier to understand
- Turned some frequently gathered data, such as
cpu vendor into class attributes, so they're determined
only once
- Separate module load, unload, and qemu killing operations
- Do not ignore module build errors
- Introduce a PreinstalledKvm class, that assumes KVM was
installed prior to KVM autotest execution
Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
---
client/tests/kvm/installer.py | 781 +++++++++++++++++++++++++++++++++++++++++
1 files changed, 781 insertions(+), 0 deletions(-)
create mode 100644 client/tests/kvm/installer.py
diff --git a/client/tests/kvm/installer.py b/client/tests/kvm/installer.py
new file mode 100644
index 0000000..8eaa554
--- /dev/null
+++ b/client/tests/kvm/installer.py
@@ -0,0 +1,781 @@
+import time, os, sys, urllib, re, signal, logging, datetime, glob, ConfigParser
+import shutil
+from autotest_lib.client.bin import utils, test, os_dep
+from autotest_lib.client.common_lib import error
+import kvm_utils
+
+
+def check_configure_options(script_path):
+ """
+ Return the list of available options (flags) of a given kvm configure build
+ script.
+
+ @param script: Path to the configure script
+ """
+ abspath = os.path.abspath(script_path)
+ help_raw = utils.system_output('%s --help' % abspath, ignore_status=True)
+ help_output = help_raw.split("\n")
+ option_list = []
+ for line in help_output:
+ cleaned_line = line.lstrip()
+ if cleaned_line.startswith("--"):
+ option = cleaned_line.split()[0]
+ option = option.split("=")[0]
+ option_list.append(option)
+
+ return option_list
+
+
+def kill_qemu_processes():
+ """
+ Kills all qemu processes, also kills all processes holding /dev/kvm down.
+ """
+ logging.debug("Killing any qemu processes that might be left behind")
+ utils.system("pkill qemu", ignore_status=True)
+ # Let's double check to see if some other process is holding /dev/kvm
+ if os.path.isfile("/dev/kvm"):
+ utils.system("fuser -k /dev/kvm", ignore_status=True)
+
+
+def cpu_vendor():
+ vendor = "intel"
+ if os.system("grep vmx /proc/cpuinfo 1>/dev/null") != 0:
+ vendor = "amd"
+ logging.debug("Detected CPU vendor as '%s'" %(vendor))
+ return vendor
+
+
+def _unload_kvm_modules(mod_list):
+ logging.info("Unloading previously loaded KVM modules")
+ for module in reversed(mod_list):
+ utils.unload_module(module)
+
+
+def _load_kvm_modules(mod_list, module_dir=None, load_stock=False):
+ """
+ Just load the KVM modules, without killing Qemu or unloading previous
+ modules.
+
+ Load modules present on any sub directory of module_dir. Function will walk
+ through module_dir until it finds the modules.
+
+ @param module_dir: Directory where the KVM modules are located.
+ @param load_stock: Whether we are going to load system kernel modules.
+ @param extra_modules: List of extra modules to load.
+ """
+ if module_dir:
+ logging.info("Loading the built KVM modules...")
+ kvm_module_path = None
+ kvm_vendor_module_path = None
+ abort = False
+
+ list_modules = ['%s.ko' % (m) for m in mod_list]
+
+ list_module_paths = []
+ for folder, subdirs, files in os.walk(module_dir):
+ for module in list_modules:
+ if module in files:
+ module_path = os.path.join(folder, module)
+ list_module_paths.append(module_path)
+
+ # We might need to arrange the modules in the correct order
+ # to avoid module load problems
+ list_modules_load = []
+ for module in list_modules:
+ for module_path in list_module_paths:
+ if os.path.basename(module_path) == module:
+ list_modules_load.append(module_path)
+
+ if len(list_module_paths) != len(list_modules):
+ logging.error("KVM modules not found. If you don't want to use the "
+ "modules built by this test, make sure the option "
+ "load_modules: 'no' is marked on the test control "
+ "file.")
+ raise error.TestError("The modules %s were requested to be loaded, "
+ "but the only modules found were %s" %
+ (list_modules, list_module_paths))
+
+ for module_path in list_modules_load:
+ try:
+ utils.system("insmod %s" % module_path)
+ except Exception, e:
+ raise error.TestFail("Failed to load KVM modules: %s" % e)
+
+ if load_stock:
+ logging.info("Loading current system KVM modules...")
+ for module in mod_list:
+ utils.system("modprobe %s" % module)
+
+
+def create_symlinks(test_bindir, prefix=None, bin_list=None, unittest=None):
+ """
+ Create symbolic links for the appropriate qemu and qemu-img commands on
+ the kvm test bindir.
+
+ @param test_bindir: KVM test bindir
+ @param prefix: KVM prefix path
+ @param bin_list: List of qemu binaries to link
+ @param unittest: Path to configuration file unittests.cfg
+ """
+ qemu_path = os.path.join(test_bindir, "qemu")
+ qemu_img_path = os.path.join(test_bindir, "qemu-img")
+ qemu_unittest_path = os.path.join(test_bindir, "unittests")
+ if os.path.lexists(qemu_path):
+ os.unlink(qemu_path)
+ if os.path.lexists(qemu_img_path):
+ os.unlink(qemu_img_path)
+ if unittest and os.path.lexists(qemu_unittest_path):
+ os.unlink(qemu_unittest_path)
+
+ logging.debug("Linking qemu binaries")
+
+ if bin_list:
+ for bin in bin_list:
+ if os.path.basename(bin) == 'qemu-kvm':
+ os.symlink(bin, qemu_path)
+ elif os.path.basename(bin) == 'qemu-img':
+ os.symlink(bin, qemu_img_path)
+
+ elif prefix:
+ kvm_qemu = os.path.join(prefix, "bin", "qemu-system-x86_64")
+ if not os.path.isfile(kvm_qemu):
+ raise error.TestError('Invalid qemu path')
+ kvm_qemu_img = os.path.join(prefix, "bin", "qemu-img")
+ if not os.path.isfile(kvm_qemu_img):
+ raise error.TestError('Invalid qemu-img path')
+ os.symlink(kvm_qemu, qemu_path)
+ os.symlink(kvm_qemu_img, qemu_img_path)
+
+ if unittest:
+ logging.debug("Linking unittest dir")
+ os.symlink(unittest, qemu_unittest_path)
+
+
+def install_roms(rom_dir, prefix):
+ logging.debug("Path to roms specified. Copying roms to install prefix")
+ rom_dst_dir = os.path.join(prefix, 'share', 'qemu')
+ for rom_src in glob.glob('%s/*.bin' % rom_dir):
+ rom_dst = os.path.join(rom_dst_dir, os.path.basename(rom_src))
+ logging.debug("Copying rom file %s to %s", rom_src, rom_dst)
+ shutil.copy(rom_src, rom_dst)
+
+
+def save_build(build_dir, dest_dir):
+ logging.debug('Saving the result of the build on %s', dest_dir)
+ base_name = os.path.basename(build_dir)
+ tarball_name = base_name + '.tar.bz2'
+ os.chdir(os.path.dirname(build_dir))
+ utils.system('tar -cjf %s %s' % (tarball_name, base_name))
+ shutil.move(tarball_name, os.path.join(dest_dir, tarball_name))
+
+
+class KvmInstallException(Exception):
+ pass
+
+
+class FailedKvmInstall(KvmInstallException):
+ pass
+
+
+class KvmNotInstalled(KvmInstallException):
+ pass
+
+
+class BaseInstaller(object):
+ # default value for load_stock argument
+ load_stock_modules = True
+ def __init__(self, mode=None):
+ self.install_mode = mode
+ self._full_module_list = None
+
+ def set_install_params(self, test, params):
+ self.params = params
+
+ load_modules = params.get('load_modules', 'no')
+ if not load_modules or load_modules == 'yes':
+ self.should_load_modules = True
+ elif load_modules == 'no':
+ self.should_load_modules = False
+ default_extra_modules = str(None)
+ self.extra_modules = eval(params.get("extra_modules",
+ default_extra_modules))
+
+ self.cpu_vendor = cpu_vendor()
+
+ self.srcdir = test.srcdir
+ if not os.path.isdir(self.srcdir):
+ os.makedirs(self.srcdir)
+
+ self.test_bindir = test.bindir
+ self.results_dir = test.resultsdir
+
+ # KVM build prefix, for the modes that do need it
+ prefix = os.path.join(test.bindir, 'build')
+ self.prefix = os.path.abspath(prefix)
+
+ # Current host kernel directory
+ default_host_kernel_source = '/lib/modules/%s/build' % os.uname()[2]
+ self.host_kernel_srcdir = params.get('host_kernel_source',
+ default_host_kernel_source)
+
+ # Extra parameters that can be passed to the configure script
+ self.extra_configure_options = params.get('extra_configure_options',
+ None)
+
+ # Do we want to save the result of the build on test.resultsdir?
+ self.save_results = True
+ save_results = params.get('save_results', 'no')
+ if save_results == 'no':
+ self.save_results = False
+
+ self._full_module_list = list(self._module_list())
+
+
+ def full_module_list(self):
+ """Return the module list used by the installer
+
+ Used by the module_probe test, to avoid using utils.unload_module().
+ """
+ if self._full_module_list is None:
+ raise KvmNotInstalled("KVM modules not installed yet (installer: %s)" % (type(self)))
+ return self._full_module_list
+
+
+ def _module_list(self):
+ """Generate the list of modules that need to be loaded
+ """
+ yield 'kvm'
+ yield 'kvm-%s' % (self.cpu_vendor)
+ if self.extra_modules:
+ for module in self.extra_modules:
+ yield module
+
+
+ def _load_modules(self, mod_list):
+ """
+ Load the KVM modules
+
+ May be overridden by subclasses.
+ """
+ _load_kvm_modules(mod_list, load_stock=self.load_stock_modules)
+
+
+ def load_modules(self, mod_list=None):
+ if mod_list is None:
+ mod_list = self.full_module_list()
+ self._load_modules(mod_list)
+
+
+ def _unload_modules(self, mod_list=None):
+ """
+ Just unload the KVM modules, without trying to kill Qemu
+ """
+ if mod_list is None:
+ mod_list = self.full_module_list()
+ _unload_kvm_modules(mod_list)
+
+
+ def unload_modules(self, mod_list=None):
+ """
+ Kill Qemu and unload the KVM modules
+ """
+ kill_qemu_processes()
+ self._unload_modules(mod_list)
+
+
+ def reload_modules(self):
+ """
+ Reload the KVM modules after killing Qemu and unloading the current modules
+ """
+ self.unload_modules()
+ self.load_modules()
+
+
+ def reload_modules_if_needed(self):
+ if self.should_load_modules:
+ self.reload_modules()
+
+
+class YumInstaller(BaseInstaller):
+ """
+ Class that uses yum to install and remove packages.
+ """
+ load_stock_modules = True
+ def set_install_params(self, test, params):
+ super(YumInstaller, self).set_install_params(test, params)
+ # Checking if all required dependencies are available
+ os_dep.command("rpm")
+ os_dep.command("yum")
+
+ default_pkg_list = str(['qemu-kvm', 'qemu-kvm-tools'])
+ default_qemu_bin_paths = str(['/usr/bin/qemu-kvm', '/usr/bin/qemu-img'])
+ default_pkg_path_list = str(None)
+ self.pkg_list = eval(params.get("pkg_list", default_pkg_list))
+ self.pkg_path_list = eval(params.get("pkg_path_list",
+ default_pkg_path_list))
+ self.qemu_bin_paths = eval(params.get("qemu_bin_paths",
+ default_qemu_bin_paths))
+
+
+ def _clean_previous_installs(self):
+ kill_qemu_processes()
+ removable_packages = ""
+ for pkg in self.pkg_list:
+ removable_packages += " %s" % pkg
+
+ utils.system("yum remove -y %s" % removable_packages)
+
+
+ def _get_packages(self):
+ for pkg in self.pkg_path_list:
+ utils.get_file(pkg, os.path.join(self.srcdir,
+ os.path.basename(pkg)))
+
+
+ def _install_packages(self):
+ """
+ Install all downloaded packages.
+ """
+ os.chdir(self.srcdir)
+ utils.system("yum install --nogpgcheck -y *.rpm")
+
+
+ def install(self):
+ self._clean_previous_installs()
+ self._get_packages()
+ self._install_packages()
+ create_symlinks(test_bindir=self.test_bindir,
+ bin_list=self.qemu_bin_paths)
+ self.reload_modules_if_needed()
+ if self.save_results:
+ save_build(self.srcdir, self.results_dir)
+
+
+class KojiInstaller(YumInstaller):
+ """
+ Class that handles installing KVM from the fedora build service, koji.
+ It uses yum to install and remove packages.
+ """
+ load_stock_modules = True
+ def set_install_params(self, test, params):
+ """
+ Gets parameters and initializes the package downloader.
+
+ @param test: kvm test object
+ @param params: Dictionary with test arguments
+ """
+ super(KojiInstaller, self).set_install_params(test, params)
+ default_koji_cmd = '/usr/bin/koji'
+ default_src_pkg = 'qemu'
+ self.src_pkg = params.get("src_pkg", default_src_pkg)
+ self.tag = params.get("koji_tag", None)
+ self.build = params.get("koji_build", None)
+ self.koji_cmd = params.get("koji_cmd", default_koji_cmd)
+
+
+ def _get_packages(self):
+ """
+ Downloads the specific arch RPMs for the specific build name.
+ """
+ downloader = kvm_utils.KojiDownloader(cmd=self.koji_cmd)
+ downloader.get(src_package=self.src_pkg, tag=self.tag,
+ build=self.build, dst_dir=self.srcdir)
+
+
+ def install(self):
+ super(KojiInstaller, self)._clean_previous_installs()
+ self._get_packages()
+ super(KojiInstaller, self)._install_packages()
+ create_symlinks(test_bindir=self.test_bindir,
+ bin_list=self.qemu_bin_paths)
+ self.reload_modules_if_needed()
+ if self.save_results:
+ save_build(self.srcdir, self.results_dir)
+
+
+class SourceDirInstaller(BaseInstaller):
+ """
+ Class that handles building/installing KVM directly from a tarball or
+ a single source code dir.
+ """
+ def set_install_params(self, test, params):
+ """
+ Initializes class attributes, and retrieves KVM code.
+
+ @param test: kvm test object
+ @param params: Dictionary with test arguments
+ """
+ super(SourceDirInstaller, self).set_install_params(test, params)
+
+ self.mod_install_dir = os.path.join(self.prefix, 'modules')
+ self.installed_kmods = False # it will be set to True in case we
+ # installed our own modules
+
+ srcdir = params.get("srcdir", None)
+ self.path_to_roms = params.get("path_to_rom_images", None)
+
+ if self.install_mode == 'localsrc':
+ if srcdir is None:
+ raise error.TestError("Install from source directory specified"
+ "but no source directory provided on the"
+ "control file.")
+ else:
+ shutil.copytree(srcdir, self.srcdir)
+
+ if self.install_mode == 'release':
+ release_tag = params.get("release_tag")
+ release_dir = params.get("release_dir")
+ release_listing = params.get("release_listing")
+ logging.info("Installing KVM from release tarball")
+ if not release_tag:
+ release_tag = kvm_utils.get_latest_kvm_release_tag(
+ release_listing)
+ tarball = os.path.join(release_dir, 'kvm', release_tag,
+ "kvm-%s.tar.gz" % release_tag)
+ logging.info("Retrieving release kvm-%s" % release_tag)
+ tarball = utils.unmap_url("/", tarball, "/tmp")
+
+ elif self.install_mode == 'snapshot':
+ logging.info("Installing KVM from snapshot")
+ snapshot_dir = params.get("snapshot_dir")
+ if not snapshot_dir:
+ raise error.TestError("Snapshot dir not provided")
+ snapshot_date = params.get("snapshot_date")
+ if not snapshot_date:
+ # Take yesterday's snapshot
+ d = (datetime.date.today() -
+ datetime.timedelta(1)).strftime("%Y%m%d")
+ else:
+ d = snapshot_date
+ tarball = os.path.join(snapshot_dir, "kvm-snapshot-%s.tar.gz" % d)
+ logging.info("Retrieving kvm-snapshot-%s" % d)
+ tarball = utils.unmap_url("/", tarball, "/tmp")
+
+ elif self.install_mode == 'localtar':
+ tarball = params.get("tarball")
+ if not tarball:
+ raise error.TestError("KVM Tarball install specified but no"
+ " tarball provided on control file.")
+ logging.info("Installing KVM from a local tarball")
+ logging.info("Using tarball %s")
+ tarball = utils.unmap_url("/", params.get("tarball"), "/tmp")
+
+ if self.install_mode in ['release', 'snapshot', 'localtar']:
+ utils.extract_tarball_to_dir(tarball, self.srcdir)
+
+ if self.install_mode in ['release', 'snapshot', 'localtar', 'srcdir']:
+ self.repo_type = kvm_utils.check_kvm_source_dir(self.srcdir)
+ configure_script = os.path.join(self.srcdir, 'configure')
+ self.configure_options = check_configure_options(configure_script)
+
+
+ def _build(self):
+ make_jobs = utils.count_cpus()
+ os.chdir(self.srcdir)
+ # For testing purposes, it's better to build qemu binaries with
+ # debugging symbols, so we can extract more meaningful stack traces.
+ cfg = "./configure --prefix=%s" % self.prefix
+ if "--disable-strip" in self.configure_options:
+ cfg += " --disable-strip"
+ steps = [cfg, "make clean", "make -j %s" % make_jobs]
+ logging.info("Building KVM")
+ for step in steps:
+ utils.system(step)
+
+
+ def _install_kmods_old_userspace(self, userspace_path):
+ """
+ Run the module install command.
+
+ This is for the "old userspace" code, that contained a 'kernel' subdirectory
+ with the kmod build code.
+
+ The code would be much simpler if we could specify the module install
+ path as parameter to the toplevel Makefile. As we can't do that and
+ the module install code doesn't use --prefix, we have to call
+ 'make -C kernel install' directly, setting the module directory
+ parameters.
+
+ If the userspace tree doens't have a 'kernel' subdirectory, the
+ module install step will be skipped.
+
+ @param userspace_path: the path the kvm-userspace directory
+ """
+ kdir = os.path.join(userspace_path, 'kernel')
+ if os.path.isdir(kdir):
+ os.chdir(kdir)
+ # INSTALLDIR is the target dir for the modules
+ # ORIGMODDIR is the dir where the old modules will be removed. we
+ # don't want to mess with the system modules, so set it
+ # to a non-existing directory
+ utils.system('make install INSTALLDIR=%s ORIGMODDIR=/tmp/no-old-modules' % (self.mod_install_dir))
+ self.installed_kmods = True
+
+
+ def _install_kmods(self, kmod_path):
+ """Run the module install command for the kmod-kvm repository
+
+ @param kmod_path: the path to the kmod-kvm.git working copy
+ """
+ os.chdir(kmod_path)
+ utils.system('make modules_install DESTDIR=%s' % (self.mod_install_dir))
+ self.installed_kmods = True
+
+
+ def _install(self):
+ os.chdir(self.srcdir)
+ logging.info("Installing KVM userspace")
+ if self.repo_type == 1:
+ utils.system("make -C qemu install")
+ self._install_kmods_old_userspace(self.srcdir)
+ elif self.repo_type == 2:
+ utils.system("make install")
+ if self.path_to_roms:
+ install_roms(self.path_to_roms, self.prefix)
+ create_symlinks(self.test_bindir, self.prefix)
+
+
+ def _load_modules(self, mod_list):
+ # load the installed KVM modules in case we installed them
+ # ourselves. Otherwise, just load the system modules.
+ if self.installed_kmods:
+ logging.info("Loading installed KVM modules")
+ _load_kvm_modules(mod_list, module_dir=self.mod_install_dir)
+ else:
+ logging.info("Loading stock KVM modules")
+ _load_kvm_modules(mod_list, load_stock=True)
+
+
+ def install(self):
+ self._build()
+ self._install()
+ self.reload_modules_if_needed()
+ if self.save_results:
+ save_build(self.srcdir, self.results_dir)
+
+
+class GitInstaller(SourceDirInstaller):
+ def _pull_code(self):
+ """
+ Retrieves code from git repositories.
+ """
+ params = self.params
+
+ kernel_repo = params.get("git_repo")
+ user_repo = params.get("user_git_repo")
+ kmod_repo = params.get("kmod_repo")
+ test_repo = params.get("test_git_repo")
+
+ kernel_branch = params.get("kernel_branch", "master")
+ user_branch = params.get("user_branch", "master")
+ kmod_branch = params.get("kmod_branch", "master")
+ test_branch = params.get("test_branch", "master")
+
+ kernel_lbranch = params.get("kernel_lbranch", "master")
+ user_lbranch = params.get("user_lbranch", "master")
+ kmod_lbranch = params.get("kmod_lbranch", "master")
+ test_lbranch = params.get("test_lbranch", "master")
+
+ kernel_commit = params.get("kernel_commit", None)
+ user_commit = params.get("user_commit", None)
+ kmod_commit = params.get("kmod_commit", None)
+ test_commit = params.get("test_commit", None)
+
+ kernel_patches = eval(params.get("kernel_patches", "[]"))
+ user_patches = eval(params.get("user_patches", "[]"))
+ kmod_patches = eval(params.get("user_patches", "[]"))
+
+ if not user_repo:
+ message = "KVM user git repository path not specified"
+ logging.error(message)
+ raise error.TestError(message)
+
+ userspace_srcdir = os.path.join(self.srcdir, "kvm_userspace")
+ kvm_utils.get_git_branch(user_repo, user_branch, userspace_srcdir,
+ user_commit, user_lbranch)
+ self.userspace_srcdir = userspace_srcdir
+
+ if user_patches:
+ os.chdir(self.userspace_srcdir)
+ for patch in user_patches:
+ utils.get_file(patch, os.path.join(self.userspace_srcdir,
+ os.path.basename(patch)))
+ utils.system('patch -p1 %s' % os.path.basename(patch))
+
+ if test_repo:
+ test_srcdir = os.path.join(self.srcdir, "kvm-unit-tests")
+ kvm_utils.get_git_branch(test_repo, test_branch, test_srcdir,
+ test_commit, test_lbranch)
+ unittest_cfg = os.path.join(test_srcdir, 'x86',
+ 'unittests.cfg')
+ self.test_srcdir = test_srcdir
+ else:
+ unittest_cfg = os.path.join(userspace_srcdir, 'kvm', 'test', 'x86',
+ 'unittests.cfg')
+
+ self.unittest_cfg = None
+ if os.path.isfile(unittest_cfg):
+ self.unittest_cfg = unittest_cfg
+
+ if kernel_repo:
+ kernel_srcdir = os.path.join(self.srcdir, "kvm")
+ kvm_utils.get_git_branch(kernel_repo, kernel_branch, kernel_srcdir,
+ kernel_commit, kernel_lbranch)
+ self.kernel_srcdir = kernel_srcdir
+ if kernel_patches:
+ os.chdir(self.kernel_srcdir)
+ for patch in kernel_patches:
+ utils.get_file(patch, os.path.join(self.userspace_srcdir,
+ os.path.basename(patch)))
+ utils.system('patch -p1 %s' % os.path.basename(patch))
+ else:
+ self.kernel_srcdir = None
+
+ if kmod_repo:
+ kmod_srcdir = os.path.join (self.srcdir, "kvm_kmod")
+ kvm_utils.get_git_branch(kmod_repo, kmod_branch, kmod_srcdir,
+ kmod_commit, kmod_lbranch)
+ self.kmod_srcdir = kmod_srcdir
+ if kmod_patches:
+ os.chdir(self.kmod_srcdir)
+ for patch in kmod_patches:
+ utils.get_file(patch, os.path.join(self.userspace_srcdir,
+ os.path.basename(patch)))
+ utils.system('patch -p1 %s' % os.path.basename(patch))
+ else:
+ self.kmod_srcdir = None
+
+ configure_script = os.path.join(self.userspace_srcdir, 'configure')
+ self.configure_options = check_configure_options(configure_script)
+
+
+ def _build(self):
+ make_jobs = utils.count_cpus()
+ cfg = './configure'
+ if self.kmod_srcdir:
+ logging.info('Building KVM modules')
+ os.chdir(self.kmod_srcdir)
+ module_build_steps = [cfg,
+ 'make clean',
+ 'make sync LINUX=%s' % self.kernel_srcdir,
+ 'make']
+ elif self.kernel_srcdir:
+ logging.info('Building KVM modules')
+ os.chdir(self.userspace_srcdir)
+ cfg += ' --kerneldir=%s' % self.host_kernel_srcdir
+ module_build_steps = [cfg,
+ 'make clean',
+ 'make -C kernel LINUX=%s sync' % self.kernel_srcdir]
+ else:
+ module_build_steps = []
+
+ for step in module_build_steps:
+ utils.run(step)
+
+ logging.info('Building KVM userspace code')
+ os.chdir(self.userspace_srcdir)
+ cfg += ' --prefix=%s' % self.prefix
+ if "--disable-strip" in self.configure_options:
+ cfg += ' --disable-strip'
+ if self.extra_configure_options:
+ cfg += ' %s' % self.extra_configure_options
+ utils.system(cfg)
+ utils.system('make clean')
+ utils.system('make -j %s' % make_jobs)
+
+ self.unittest_prefix = None
+ if self.unittest_cfg:
+ os.chdir(os.path.dirname(os.path.dirname(self.unittest_cfg)))
+ utils.system('./configure --prefix=%s' % self.prefix)
+ utils.system('make')
+ utils.system('make install')
+ self.unittest_prefix = os.path.join(self.prefix, 'share', 'qemu',
+ 'tests')
+
+
+ def _install(self):
+ if self.kernel_srcdir:
+ os.chdir(self.userspace_srcdir)
+ # the kernel module install with --prefix doesn't work, and DESTDIR
+ # wouldn't work for the userspace stuff, so we clear WANT_MODULE:
+ utils.system('make install WANT_MODULE=')
+ # and install the old-style-kmod modules manually:
+ self._install_kmods_old_userspace(self.userspace_srcdir)
+ elif self.kmod_srcdir:
+ # if we have a kmod repository, it is easier:
+ # 1) install userspace:
+ os.chdir(self.userspace_srcdir)
+ utils.system('make install')
+ # 2) install kmod:
+ self._install_kmods(self.kmod_srcdir)
+ else:
+ # if we don't have kmod sources, we just install
+ # userspace:
+ os.chdir(self.userspace_srcdir)
+ utils.system('make install')
+
+ if self.path_to_roms:
+ install_roms(self.path_to_roms, self.prefix)
+ create_symlinks(test_bindir=self.test_bindir, prefix=self.prefix,
+ bin_list=None,
+ unittest=self.unittest_prefix)
+
+
+ def install(self):
+ self._pull_code()
+ self._build()
+ self._install()
+ self.reload_modules_if_needed()
+ if self.save_results:
+ save_build(self.srcdir, self.results_dir)
+
+
+class PreInstalledKvm(BaseInstaller):
+ # load_modules() will use the stock modules:
+ load_stock_modules = True
+ def install(self):
+ logging.info("Expecting KVM to be already installed. Doing nothing")
+
+
+class FailedInstaller:
+ """
+ Class used to be returned instead of the installer if a installation fails
+
+ Useful to make sure no installer object is used if KVM installation fails.
+ """
+ def __init__(self, msg="KVM install failed"):
+ self._msg = msg
+
+
+ def load_modules(self):
+ """Will refuse to load the KVM modules as install failed"""
+ raise FailedKvmInstall("KVM modules not available. reason: %s" % (self._msg))
+
+
+installer_classes = {
+ 'localsrc': SourceDirInstaller,
+ 'localtar': SourceDirInstaller,
+ 'release': SourceDirInstaller,
+ 'snapshot': SourceDirInstaller,
+ 'git': GitInstaller,
+ 'yum': YumInstaller,
+ 'koji': KojiInstaller,
+ 'preinstalled': PreInstalledKvm,
+}
+
+
+def _installer_class(install_mode):
+ c = installer_classes.get(install_mode)
+ if c is None:
+ raise error.TestError('Invalid or unsupported'
+ ' install mode: %s' % install_mode)
+ return c
+
+
+def make_installer(params):
+ # priority:
+ # - 'install_mode' param
+ # - 'mode' param
+ mode = params.get("install_mode", params.get("mode"))
+ klass = _installer_class(mode)
+ return klass(mode)
--
1.7.2.3
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 2/8] KVM test: rename 'mode' to 'install_mode' on the sample cfg files
2010-12-29 22:37 [PATCH 0/8] KVM test: Build code refactor + module_probe test Lucas Meneghel Rodrigues
2010-12-29 22:37 ` [PATCH 1/8] KVM test: Refactor the KVM build code into installer Lucas Meneghel Rodrigues
@ 2010-12-29 22:37 ` Lucas Meneghel Rodrigues
2010-12-29 22:37 ` [PATCH 3/8] KVM test: add methods to store a installer object in 'env' Lucas Meneghel Rodrigues
` (5 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Lucas Meneghel Rodrigues @ 2010-12-29 22:37 UTC (permalink / raw)
To: autotest; +Cc: Eduardo Habkost, kvm
From: Eduardo Habkost <ehabkost@redhat.com>
The old name still works, but the new name is clearer.
Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
---
client/tests/kvm/build.cfg.sample | 14 +++++++-------
1 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/client/tests/kvm/build.cfg.sample b/client/tests/kvm/build.cfg.sample
index b6a20f7..a689ed4 100644
--- a/client/tests/kvm/build.cfg.sample
+++ b/client/tests/kvm/build.cfg.sample
@@ -16,7 +16,7 @@ variants:
save_results = no
variants:
- release:
- mode = release
+ install_mode = release
## Install from a kvm release. You can optionally specify
## a release tag. If you omit it, the build test will get
## the latest release tag available at that moment.
@@ -27,7 +27,7 @@ variants:
# files can be copied from there to your source based install
# path_to_rom_images = /usr/share/kvm
- snapshot:
- mode = snapshot
+ install_mode = snapshot
## Install from a kvm snapshot location. You can optionally
## specify a snapshot date. If you omit it, the test will get
## yesterday's snapshot.
@@ -37,21 +37,21 @@ variants:
# files can be copied from there to your source based install
# path_to_rom_images = /usr/share/kvm
- localtar:
- mode = localtar
+ install_mode = localtar
## Install from tarball located on the host's filesystem.
tarball = /tmp/kvm-84.tar.gz
# In some cases, you might want to provide a ROM dir, so ROM
# files can be copied from there to your source based install
# path_to_rom_images = /usr/share/kvm
- localsrc:
- mode = localsrc
+ install_mode = localsrc
## Install from tarball located on the host's filesystem.
srcdir = /tmp/kvm-84
# In some cases, you might want to provide a ROM dir, so ROM
# files can be copied from there to your source based install
# path_to_rom_images = /usr/share/kvm
- git:
- mode = git
+ install_mode = git
## Install KVM from git repositories.
## If you provide only "git_repo" and "user_git_repo", the
## build test will assume it will perform all build from the
@@ -80,7 +80,7 @@ variants:
# files can be copied from there to your source based install
# path_to_rom_images = /usr/share/kvm
- yum:
- mode = yum
+ install_mode = yum
src_pkg = qemu
## Name of the rpms we need installed
pkg_list = ['qemu-kvm', 'qemu-kvm-tools', 'qemu-system-x86', 'qemu-common', 'qemu-img']
@@ -89,7 +89,7 @@ variants:
## List of RPMs that will be installed
pkg_path_list = ['http://foo.com/rpm1', 'http://foo.com/rpm2']
- koji:
- mode = koji
+ install_mode = koji
## Install KVM from koji (Fedora build server)
## It is possible to install packages right from Koji if you
## provide a release tag or a build.
--
1.7.2.3
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 3/8] KVM test: add methods to store a installer object in 'env'
2010-12-29 22:37 [PATCH 0/8] KVM test: Build code refactor + module_probe test Lucas Meneghel Rodrigues
2010-12-29 22:37 ` [PATCH 1/8] KVM test: Refactor the KVM build code into installer Lucas Meneghel Rodrigues
2010-12-29 22:37 ` [PATCH 2/8] KVM test: rename 'mode' to 'install_mode' on the sample cfg files Lucas Meneghel Rodrigues
@ 2010-12-29 22:37 ` Lucas Meneghel Rodrigues
2010-12-29 22:37 ` [PATCH 4/8] KVM test: Make the build test to use the installer library Lucas Meneghel Rodrigues
` (4 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Lucas Meneghel Rodrigues @ 2010-12-29 22:37 UTC (permalink / raw)
To: autotest; +Cc: Eduardo Habkost, kvm
This way, tests can get information about Qemu and the modules that were
installed by the 'build' test if needed.
Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
---
client/tests/kvm/kvm_utils.py | 18 ++++++++++++++++++
1 files changed, 18 insertions(+), 0 deletions(-)
diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_utils.py
index 6362490..1967ddd 100644
--- a/client/tests/kvm/kvm_utils.py
+++ b/client/tests/kvm/kvm_utils.py
@@ -121,6 +121,24 @@ class Env(UserDict.IterableUserDict):
del self["vm__%s" % name]
+ def register_installer(self, installer):
+ """
+ Register a installer that was just run
+
+ The installer will be available for other tests, so that
+ information about the installed KVM modules and qemu-kvm can be used by
+ them.
+ """
+ self['last_installer'] = installer
+
+
+ def previous_installer(self):
+ """
+ Return the last installer that was registered
+ """
+ return self.get('last_installer')
+
+
class Params(UserDict.IterableUserDict):
"""
A dict-like object passed to every test.
--
1.7.2.3
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 4/8] KVM test: Make the build test to use the installer library
2010-12-29 22:37 [PATCH 0/8] KVM test: Build code refactor + module_probe test Lucas Meneghel Rodrigues
` (2 preceding siblings ...)
2010-12-29 22:37 ` [PATCH 3/8] KVM test: add methods to store a installer object in 'env' Lucas Meneghel Rodrigues
@ 2010-12-29 22:37 ` Lucas Meneghel Rodrigues
2010-12-29 22:37 ` [PATCH 5/8] KVM test: installer: fix the 'layout 1' detection Lucas Meneghel Rodrigues
` (3 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Lucas Meneghel Rodrigues @ 2010-12-29 22:37 UTC (permalink / raw)
To: autotest; +Cc: kvm
Use the code on the new installer library and get rid
of the original code.
Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
client/tests/kvm/tests/build.py | 627 +--------------------------------------
1 files changed, 12 insertions(+), 615 deletions(-)
diff --git a/client/tests/kvm/tests/build.py b/client/tests/kvm/tests/build.py
index bb3e2dc..1eef7a1 100644
--- a/client/tests/kvm/tests/build.py
+++ b/client/tests/kvm/tests/build.py
@@ -1,604 +1,4 @@
-import time, os, sys, urllib, re, signal, logging, datetime, glob, ConfigParser
-import shutil
-from autotest_lib.client.bin import utils, test, os_dep
-from autotest_lib.client.common_lib import error
-import kvm_utils
-
-
-def check_configure_options(script_path):
- """
- Return the list of available options (flags) of a given kvm configure build
- script.
-
- @param script: Path to the configure script
- """
- abspath = os.path.abspath(script_path)
- help_raw = utils.system_output('%s --help' % abspath, ignore_status=True)
- help_output = help_raw.split("\n")
- option_list = []
- for line in help_output:
- cleaned_line = line.lstrip()
- if cleaned_line.startswith("--"):
- option = cleaned_line.split()[0]
- option = option.split("=")[0]
- option_list.append(option)
-
- return option_list
-
-
-def kill_qemu_processes():
- """
- Kills all qemu processes, also kills all processes holding /dev/kvm down.
- """
- logging.debug("Killing any qemu processes that might be left behind")
- utils.system("pkill qemu", ignore_status=True)
- # Let's double check to see if some other process is holding /dev/kvm
- if os.path.isfile("/dev/kvm"):
- utils.system("fuser -k /dev/kvm", ignore_status=True)
-
-
-def load_kvm_modules(module_dir=None, load_stock=False, extra_modules=None):
- """
- Unload previously loaded kvm modules, then load modules present on any
- sub directory of module_dir. Function will walk through module_dir until
- it finds the modules.
-
- @param module_dir: Directory where the KVM modules are located.
- @param load_stock: Whether we are going to load system kernel modules.
- @param extra_modules: List of extra modules to load.
- """
- vendor = "intel"
- if os.system("grep vmx /proc/cpuinfo 1>/dev/null") != 0:
- vendor = "amd"
- logging.debug("Detected CPU vendor as '%s'" %(vendor))
-
- kill_qemu_processes()
-
- logging.info("Unloading previously loaded KVM modules")
- utils.unload_module("kvm")
- if extra_modules:
- for module in extra_modules:
- utils.unload_module(module)
-
- if module_dir:
- logging.info("Loading the built KVM modules...")
- kvm_module_path = None
- kvm_vendor_module_path = None
- abort = False
-
- list_modules = ['kvm.ko', 'kvm-%s.ko' % vendor]
- if extra_modules:
- for extra_module in extra_modules:
- list_modules.append('%s.ko' % extra_module)
-
- list_module_paths = []
- for folder, subdirs, files in os.walk(module_dir):
- for module in list_modules:
- if module in files:
- module_path = os.path.join(folder, module)
- list_module_paths.append(module_path)
-
- # We might need to arrange the modules in the correct order
- # to avoid module load problems
- list_modules_load = []
- for module in list_modules:
- for module_path in list_module_paths:
- if os.path.basename(module_path) == module:
- list_modules_load.append(module_path)
-
- if len(list_module_paths) != len(list_modules):
- logging.error("KVM modules not found. If you don't want to use the "
- "modules built by this test, make sure the option "
- "load_modules: 'no' is marked on the test control "
- "file.")
- raise error.TestError("The modules %s were requested to be loaded, "
- "but the only modules found were %s" %
- (list_modules, list_module_paths))
-
- for module_path in list_modules_load:
- try:
- utils.system("insmod %s" % module_path)
- except Exception, e:
- raise error.TestFail("Failed to load KVM modules: %s" % e)
-
- if load_stock:
- logging.info("Loading current system KVM modules...")
- utils.system("modprobe kvm")
- utils.system("modprobe kvm-%s" % vendor)
- if extra_modules:
- for module in extra_modules:
- utils.system("modprobe %s" % module)
-
-
-def create_symlinks(test_bindir, prefix=None, bin_list=None, unittest=None):
- """
- Create symbolic links for the appropriate qemu and qemu-img commands on
- the kvm test bindir.
-
- @param test_bindir: KVM test bindir
- @param prefix: KVM prefix path
- @param bin_list: List of qemu binaries to link
- @param unittest: Path to configuration file unittests.cfg
- """
- qemu_path = os.path.join(test_bindir, "qemu")
- qemu_img_path = os.path.join(test_bindir, "qemu-img")
- qemu_unittest_path = os.path.join(test_bindir, "unittests")
- if os.path.lexists(qemu_path):
- os.unlink(qemu_path)
- if os.path.lexists(qemu_img_path):
- os.unlink(qemu_img_path)
- if unittest and os.path.lexists(qemu_unittest_path):
- os.unlink(qemu_unittest_path)
-
- logging.debug("Linking qemu binaries")
-
- if bin_list:
- for bin in bin_list:
- if os.path.basename(bin) == 'qemu-kvm':
- os.symlink(bin, qemu_path)
- elif os.path.basename(bin) == 'qemu-img':
- os.symlink(bin, qemu_img_path)
-
- elif prefix:
- kvm_qemu = os.path.join(prefix, "bin", "qemu-system-x86_64")
- if not os.path.isfile(kvm_qemu):
- raise error.TestError('Invalid qemu path')
- kvm_qemu_img = os.path.join(prefix, "bin", "qemu-img")
- if not os.path.isfile(kvm_qemu_img):
- raise error.TestError('Invalid qemu-img path')
- os.symlink(kvm_qemu, qemu_path)
- os.symlink(kvm_qemu_img, qemu_img_path)
-
- if unittest:
- logging.debug("Linking unittest dir")
- os.symlink(unittest, qemu_unittest_path)
-
-
-def install_roms(rom_dir, prefix):
- logging.debug("Path to roms specified. Copying roms to install prefix")
- rom_dst_dir = os.path.join(prefix, 'share', 'qemu')
- for rom_src in glob.glob('%s/*.bin' % rom_dir):
- rom_dst = os.path.join(rom_dst_dir, os.path.basename(rom_src))
- logging.debug("Copying rom file %s to %s", rom_src, rom_dst)
- shutil.copy(rom_src, rom_dst)
-
-
-def save_build(build_dir, dest_dir):
- logging.debug('Saving the result of the build on %s', dest_dir)
- base_name = os.path.basename(build_dir)
- tarball_name = base_name + '.tar.bz2'
- os.chdir(os.path.dirname(build_dir))
- utils.system('tar -cjf %s %s' % (tarball_name, base_name))
- shutil.move(tarball_name, os.path.join(dest_dir, tarball_name))
-
-
-class BaseInstaller(object):
- def __init__(self, test, params):
- load_modules = params.get('load_modules', 'no')
- if not load_modules or load_modules == 'yes':
- self.load_modules = True
- elif load_modules == 'no':
- self.load_modules = False
- default_extra_modules = str(None)
- self.extra_modules = eval(params.get("extra_modules",
- default_extra_modules))
-
- self.srcdir = test.srcdir
- if not os.path.isdir(self.srcdir):
- os.makedirs(self.srcdir)
-
- self.test_bindir = test.bindir
- self.results_dir = test.resultsdir
-
- # KVM build prefix, for the modes that do need it
- prefix = os.path.join(test.bindir, 'build')
- self.prefix = os.path.abspath(prefix)
-
- # Current host kernel directory
- default_host_kernel_source = '/lib/modules/%s/build' % os.uname()[2]
- self.host_kernel_srcdir = params.get('host_kernel_source',
- default_host_kernel_source)
-
- # Extra parameters that can be passed to the configure script
- self.extra_configure_options = params.get('extra_configure_options',
- None)
-
- # Do we want to save the result of the build on test.resultsdir?
- self.save_results = True
- save_results = params.get('save_results', 'no')
- if save_results == 'no':
- self.save_results = False
-
-
-class YumInstaller(BaseInstaller):
- """
- Class that uses yum to install and remove packages.
- """
- def __init__(self, test, params):
- super(YumInstaller, self).__init__(test, params)
- # Checking if all required dependencies are available
- os_dep.command("rpm")
- os_dep.command("yum")
-
- default_pkg_list = str(['qemu-kvm', 'qemu-kvm-tools'])
- default_qemu_bin_paths = str(['/usr/bin/qemu-kvm', '/usr/bin/qemu-img'])
- default_pkg_path_list = str(None)
- self.pkg_list = eval(params.get("pkg_list", default_pkg_list))
- self.pkg_path_list = eval(params.get("pkg_path_list",
- default_pkg_path_list))
- self.qemu_bin_paths = eval(params.get("qemu_bin_paths",
- default_qemu_bin_paths))
-
-
- def _clean_previous_installs(self):
- kill_qemu_processes()
- removable_packages = ""
- for pkg in self.pkg_list:
- removable_packages += " %s" % pkg
-
- utils.system("yum remove -y %s" % removable_packages)
-
-
- def _get_packages(self):
- for pkg in self.pkg_path_list:
- utils.get_file(pkg, os.path.join(self.srcdir,
- os.path.basename(pkg)))
-
-
- def _install_packages(self):
- """
- Install all downloaded packages.
- """
- os.chdir(self.srcdir)
- utils.system("yum install --nogpgcheck -y *.rpm")
-
-
- def install(self):
- self._clean_previous_installs()
- self._get_packages()
- self._install_packages()
- create_symlinks(test_bindir=self.test_bindir,
- bin_list=self.qemu_bin_paths)
- if self.load_modules:
- load_kvm_modules(load_stock=True, extra_modules=self.extra_modules)
- if self.save_results:
- save_build(self.srcdir, self.results_dir)
-
-
-class KojiInstaller(YumInstaller):
- """
- Class that handles installing KVM from the fedora build service, koji.
- It uses yum to install and remove packages.
- """
- def __init__(self, test, params):
- """
- Gets parameters and initializes the package downloader.
-
- @param test: kvm test object
- @param params: Dictionary with test arguments
- """
- super(KojiInstaller, self).__init__(test, params)
- default_koji_cmd = '/usr/bin/koji'
- default_src_pkg = 'qemu'
- self.src_pkg = params.get("src_pkg", default_src_pkg)
- self.tag = params.get("koji_tag", None)
- self.build = params.get("koji_build", None)
- koji_cmd = params.get("koji_cmd", default_koji_cmd)
- self.downloader = kvm_utils.KojiDownloader(cmd=koji_cmd)
-
-
- def _get_packages(self):
- """
- Downloads the specific arch RPMs for the specific build name.
- """
- self.downloader.get(src_package=self.src_pkg, tag=self.tag,
- build=self.build, dst_dir=self.srcdir)
-
-
- def install(self):
- super(KojiInstaller, self)._clean_previous_installs()
- self._get_packages()
- super(KojiInstaller, self)._install_packages()
- create_symlinks(test_bindir=self.test_bindir,
- bin_list=self.qemu_bin_paths)
- if self.load_modules:
- load_kvm_modules(load_stock=True, extra_modules=self.extra_modules)
- if self.save_results:
- save_build(self.srcdir, self.results_dir)
-
-
-class SourceDirInstaller(BaseInstaller):
- """
- Class that handles building/installing KVM directly from a tarball or
- a single source code dir.
- """
- def __init__(self, test, params):
- """
- Initializes class attributes, and retrieves KVM code.
-
- @param test: kvm test object
- @param params: Dictionary with test arguments
- """
- super(SourceDirInstaller, self).__init__(test, params)
-
- install_mode = params["mode"]
- srcdir = params.get("srcdir", None)
- self.path_to_roms = params.get("path_to_rom_images", None)
-
- if install_mode == 'localsrc':
- if srcdir is None:
- raise error.TestError("Install from source directory specified"
- "but no source directory provided on the"
- "control file.")
- else:
- shutil.copytree(srcdir, self.srcdir)
-
- if install_mode == 'release':
- release_tag = params.get("release_tag")
- release_dir = params.get("release_dir")
- release_listing = params.get("release_listing")
- logging.info("Installing KVM from release tarball")
- if not release_tag:
- release_tag = kvm_utils.get_latest_kvm_release_tag(
- release_listing)
- tarball = os.path.join(release_dir, 'kvm', release_tag,
- "kvm-%s.tar.gz" % release_tag)
- logging.info("Retrieving release kvm-%s" % release_tag)
- tarball = utils.unmap_url("/", tarball, "/tmp")
-
- elif install_mode == 'snapshot':
- logging.info("Installing KVM from snapshot")
- snapshot_dir = params.get("snapshot_dir")
- if not snapshot_dir:
- raise error.TestError("Snapshot dir not provided")
- snapshot_date = params.get("snapshot_date")
- if not snapshot_date:
- # Take yesterday's snapshot
- d = (datetime.date.today() -
- datetime.timedelta(1)).strftime("%Y%m%d")
- else:
- d = snapshot_date
- tarball = os.path.join(snapshot_dir, "kvm-snapshot-%s.tar.gz" % d)
- logging.info("Retrieving kvm-snapshot-%s" % d)
- tarball = utils.unmap_url("/", tarball, "/tmp")
-
- elif install_mode == 'localtar':
- tarball = params.get("tarball")
- if not tarball:
- raise error.TestError("KVM Tarball install specified but no"
- " tarball provided on control file.")
- logging.info("Installing KVM from a local tarball")
- logging.info("Using tarball %s")
- tarball = utils.unmap_url("/", params.get("tarball"), "/tmp")
-
- if install_mode in ['release', 'snapshot', 'localtar']:
- utils.extract_tarball_to_dir(tarball, self.srcdir)
-
- if install_mode in ['release', 'snapshot', 'localtar', 'srcdir']:
- self.repo_type = kvm_utils.check_kvm_source_dir(self.srcdir)
- configure_script = os.path.join(self.srcdir, 'configure')
- self.configure_options = check_configure_options(configure_script)
-
-
- def _build(self):
- make_jobs = utils.count_cpus()
- os.chdir(self.srcdir)
- # For testing purposes, it's better to build qemu binaries with
- # debugging symbols, so we can extract more meaningful stack traces.
- cfg = "./configure --prefix=%s" % self.prefix
- if "--disable-strip" in self.configure_options:
- cfg += " --disable-strip"
- steps = [cfg, "make clean", "make -j %s" % make_jobs]
- logging.info("Building KVM")
- for step in steps:
- utils.system(step)
-
-
- def _install(self):
- os.chdir(self.srcdir)
- logging.info("Installing KVM userspace")
- if self.repo_type == 1:
- utils.system("make -C qemu install")
- elif self.repo_type == 2:
- utils.system("make install")
- if self.path_to_roms:
- install_roms(self.path_to_roms, self.prefix)
- create_symlinks(self.test_bindir, self.prefix)
-
-
- def _load_modules(self):
- load_kvm_modules(module_dir=self.srcdir,
- extra_modules=self.extra_modules)
-
-
- def install(self):
- self._build()
- self._install()
- if self.load_modules:
- self._load_modules()
- if self.save_results:
- save_build(self.srcdir, self.results_dir)
-
-
-class GitInstaller(SourceDirInstaller):
- def __init__(self, test, params):
- """
- Initialize class parameters and retrieves code from git repositories.
-
- @param test: kvm test object.
- @param params: Dictionary with test parameters.
- """
- super(GitInstaller, self).__init__(test, params)
-
- kernel_repo = params.get("git_repo")
- user_repo = params.get("user_git_repo")
- kmod_repo = params.get("kmod_repo")
- test_repo = params.get("test_git_repo")
-
- kernel_branch = params.get("kernel_branch", "master")
- user_branch = params.get("user_branch", "master")
- kmod_branch = params.get("kmod_branch", "master")
- test_branch = params.get("test_branch", "master")
-
- kernel_lbranch = params.get("kernel_lbranch", "master")
- user_lbranch = params.get("user_lbranch", "master")
- kmod_lbranch = params.get("kmod_lbranch", "master")
- test_lbranch = params.get("test_lbranch", "master")
-
- kernel_commit = params.get("kernel_commit", None)
- user_commit = params.get("user_commit", None)
- kmod_commit = params.get("kmod_commit", None)
- test_commit = params.get("test_commit", None)
-
- kernel_patches = eval(params.get("kernel_patches", "[]"))
- user_patches = eval(params.get("user_patches", "[]"))
- kmod_patches = eval(params.get("user_patches", "[]"))
-
- if not user_repo:
- message = "KVM user git repository path not specified"
- logging.error(message)
- raise error.TestError(message)
-
- userspace_srcdir = os.path.join(self.srcdir, "kvm_userspace")
- kvm_utils.get_git_branch(user_repo, user_branch, userspace_srcdir,
- user_commit, user_lbranch)
- self.userspace_srcdir = userspace_srcdir
-
- if user_patches:
- os.chdir(self.userspace_srcdir)
- for patch in user_patches:
- utils.get_file(patch, os.path.join(self.userspace_srcdir,
- os.path.basename(patch)))
- utils.system('patch -p1 %s' % os.path.basename(patch))
-
- if test_repo:
- test_srcdir = os.path.join(self.srcdir, "kvm-unit-tests")
- kvm_utils.get_git_branch(test_repo, test_branch, test_srcdir,
- test_commit, test_lbranch)
- unittest_cfg = os.path.join(test_srcdir, 'x86',
- 'unittests.cfg')
- self.test_srcdir = test_srcdir
- else:
- unittest_cfg = os.path.join(userspace_srcdir, 'kvm', 'test', 'x86',
- 'unittests.cfg')
-
- self.unittest_cfg = None
- if os.path.isfile(unittest_cfg):
- self.unittest_cfg = unittest_cfg
-
- if kernel_repo:
- kernel_srcdir = os.path.join(self.srcdir, "kvm")
- kvm_utils.get_git_branch(kernel_repo, kernel_branch, kernel_srcdir,
- kernel_commit, kernel_lbranch)
- self.kernel_srcdir = kernel_srcdir
- if kernel_patches:
- os.chdir(self.kernel_srcdir)
- for patch in kernel_patches:
- utils.get_file(patch, os.path.join(self.userspace_srcdir,
- os.path.basename(patch)))
- utils.system('patch -p1 %s' % os.path.basename(patch))
- else:
- self.kernel_srcdir = None
-
- if kmod_repo:
- kmod_srcdir = os.path.join (self.srcdir, "kvm_kmod")
- kvm_utils.get_git_branch(kmod_repo, kmod_branch, kmod_srcdir,
- kmod_commit, kmod_lbranch)
- self.kmod_srcdir = kmod_srcdir
- if kmod_patches:
- os.chdir(self.kmod_srcdir)
- for patch in kmod_patches:
- utils.get_file(patch, os.path.join(self.userspace_srcdir,
- os.path.basename(patch)))
- utils.system('patch -p1 %s' % os.path.basename(patch))
- else:
- self.kmod_srcdir = None
-
- configure_script = os.path.join(self.userspace_srcdir, 'configure')
- self.configure_options = check_configure_options(configure_script)
-
-
- def _build(self):
- make_jobs = utils.count_cpus()
- cfg = './configure'
- self.modules_build_succeed = False
- if self.kmod_srcdir:
- logging.info('Building KVM modules')
- os.chdir(self.kmod_srcdir)
- module_build_steps = [cfg,
- 'make clean',
- 'make sync LINUX=%s' % self.kernel_srcdir,
- 'make']
- elif self.kernel_srcdir:
- logging.info('Building KVM modules')
- os.chdir(self.userspace_srcdir)
- cfg += ' --kerneldir=%s' % self.host_kernel_srcdir
- module_build_steps = [cfg,
- 'make clean',
- 'make -C kernel LINUX=%s sync' % self.kernel_srcdir]
- else:
- module_build_steps = []
-
- try:
- if module_build_steps:
- for step in module_build_steps:
- utils.run(step)
- self.modules_build_succeed = True
- except error.CmdError, e:
- logging.error("KVM modules build failed to build: %s" % e)
-
- logging.info('Building KVM userspace code')
- os.chdir(self.userspace_srcdir)
- cfg += ' --prefix=%s' % self.prefix
- if "--disable-strip" in self.configure_options:
- cfg += ' --disable-strip'
- if self.extra_configure_options:
- cfg += ' %s' % self.extra_configure_options
- utils.system(cfg)
- utils.system('make clean')
- utils.system('make -j %s' % make_jobs)
-
- self.unittest_prefix = None
- if self.unittest_cfg:
- os.chdir(os.path.dirname(os.path.dirname(self.unittest_cfg)))
- utils.system('./configure --prefix=%s' % self.prefix)
- utils.system('make')
- utils.system('make install')
- self.unittest_prefix = os.path.join(self.prefix, 'share', 'qemu',
- 'tests')
-
-
- def _install(self):
- os.chdir(self.userspace_srcdir)
- utils.system('make install')
- if self.path_to_roms:
- install_roms(self.path_to_roms, self.prefix)
- create_symlinks(test_bindir=self.test_bindir, prefix=self.prefix,
- bin_list=None,
- unittest=self.unittest_prefix)
-
-
- def _load_modules(self):
- if self.kmod_srcdir and self.modules_build_succeed:
- load_kvm_modules(module_dir=self.kmod_srcdir,
- extra_modules=self.extra_modules)
- elif self.kernel_srcdir and self.modules_build_succeed:
- load_kvm_modules(module_dir=self.userspace_srcdir,
- extra_modules=self.extra_modules)
- else:
- logging.info("Loading stock KVM modules")
- load_kvm_modules(load_stock=True,
- extra_modules=self.extra_modules)
-
-
- def install(self):
- self._build()
- self._install()
- if self.load_modules:
- self._load_modules()
- if self.save_results:
- save_build(self.srcdir, self.results_dir)
-
+import installer
def run_build(test, params, env):
"""
@@ -609,20 +9,17 @@ def run_build(test, params, env):
@param params: Dictionary with test parameters.
@param env: Test environment.
"""
- install_mode = params.get("mode")
srcdir = params.get("srcdir", test.srcdir)
params["srcdir"] = srcdir
- if install_mode in ['localsrc', 'localtar', 'release', 'snapshot']:
- installer = SourceDirInstaller(test, params)
- elif install_mode == 'git':
- installer = GitInstaller(test, params)
- elif install_mode == 'yum':
- installer = YumInstaller(test, params)
- elif install_mode == 'koji':
- installer = KojiInstaller(test, params)
- else:
- raise error.TestError('Invalid or unsupported'
- ' install mode: %s' % install_mode)
-
- installer.install()
+ try:
+ installer_object = installer.make_installer(params)
+ installer_object.set_install_params(test, params)
+ installer_object.install()
+ env.register_installer(installer_object)
+ except Exception,e:
+ # if the build/install fails, don't allow other tests
+ # to get a installer.
+ msg = "KVM install failed: %s" % (e)
+ env.register_installer(installer.FailedInstaller(msg))
+ raise
--
1.7.2.3
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 5/8] KVM test: installer: fix the 'layout 1' detection
2010-12-29 22:37 [PATCH 0/8] KVM test: Build code refactor + module_probe test Lucas Meneghel Rodrigues
` (3 preceding siblings ...)
2010-12-29 22:37 ` [PATCH 4/8] KVM test: Make the build test to use the installer library Lucas Meneghel Rodrigues
@ 2010-12-29 22:37 ` Lucas Meneghel Rodrigues
2010-12-29 22:37 ` [PATCH 6/8] KVM test: load/unload kvm module Lucas Meneghel Rodrigues
` (2 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Lucas Meneghel Rodrigues @ 2010-12-29 22:37 UTC (permalink / raw)
To: autotest; +Cc: Eduardo Habkost, kvm
From: Eduardo Habkost <ehabkost@redhat.com>
The common (or only) case where we have a 'qemu' directory (old KVM
versions) also contains a 'kvm' directory, so the detection of the 'old
layout' code was not working.
Fix this by removing the 'and not has_kvm_dir' condition.
Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
---
client/tests/kvm/kvm_utils.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_utils.py
index 1967ddd..d135979 100644
--- a/client/tests/kvm/kvm_utils.py
+++ b/client/tests/kvm/kvm_utils.py
@@ -445,7 +445,7 @@ def check_kvm_source_dir(source_dir):
os.chdir(source_dir)
has_qemu_dir = os.path.isdir('qemu')
has_kvm_dir = os.path.isdir('kvm')
- if has_qemu_dir and not has_kvm_dir:
+ if has_qemu_dir:
logging.debug("qemu directory detected, source dir layout 1")
return 1
if has_kvm_dir and not has_qemu_dir:
--
1.7.2.3
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 6/8] KVM test: load/unload kvm module
2010-12-29 22:37 [PATCH 0/8] KVM test: Build code refactor + module_probe test Lucas Meneghel Rodrigues
` (4 preceding siblings ...)
2010-12-29 22:37 ` [PATCH 5/8] KVM test: installer: fix the 'layout 1' detection Lucas Meneghel Rodrigues
@ 2010-12-29 22:37 ` Lucas Meneghel Rodrigues
2010-12-29 22:37 ` [PATCH 7/8] KVM test: add interface to get modules from cfg file, and load/unload them Lucas Meneghel Rodrigues
2010-12-29 22:37 ` [PATCH 8/8] KVM test: test/module_probe: use installer object to load modules Lucas Meneghel Rodrigues
7 siblings, 0 replies; 9+ messages in thread
From: Lucas Meneghel Rodrigues @ 2010-12-29 22:37 UTC (permalink / raw)
To: autotest; +Cc: Suqin Huang, kvm
From: Suqin Huang <shuang@redhat.com>
check whether kvm modules exist, unload them if exist, else load them.
Signed-off-by: Suqin Huang <shuang@redhat.com>
---
client/tests/kvm/tests/module_probe.py | 53 ++++++++++++++++++++++++++++++++
client/tests/kvm/tests_base.cfg.sample | 6 +++
2 files changed, 59 insertions(+), 0 deletions(-)
create mode 100644 client/tests/kvm/tests/module_probe.py
diff --git a/client/tests/kvm/tests/module_probe.py b/client/tests/kvm/tests/module_probe.py
new file mode 100644
index 0000000..519a916
--- /dev/null
+++ b/client/tests/kvm/tests/module_probe.py
@@ -0,0 +1,53 @@
+import re, commands, logging, os
+from autotest_lib.client.common_lib import error
+import kvm_subprocess, kvm_test_utils, kvm_utils
+
+def run_module_probe(test, params, env):
+ """
+ load/unload kvm modules several times.
+
+ Packet Loss Test:
+ 1) check host cpu module
+ 2) get module info
+ 3) unload modules if they exist, else load them
+
+ @param test: Kvm test object
+ @param params: Dictionary with the test parameters
+ @param env: Dictionary with test environment.
+ """
+
+ def module_probe(name_list, arg=""):
+ for name in name_list:
+ cmd = "modprobe %s %s" % (arg, name)
+ logging.debug(cmd)
+ s, o = commands.getstatusoutput(cmd)
+ if s != 0:
+ logging.error("Failed to load/unload modules %s" % o)
+ return False
+ return True
+
+ #Check host cpu module
+ flags = file("/proc/cpuinfo").read()
+ arch_check = re.findall("%s\s" % "vmx", flags)
+ if arch_check:
+ arch = "kvm_intel"
+ else:
+ arch = "kvm_amd"
+
+ #Check whether ksm module exist
+ if os.path.exists("/sys/module/ksm"):
+ mod_list = ["ksm", arch, "kvm"]
+ else:
+ mod_list = [arch, "kvm"]
+
+ logging.debug(mod_list)
+ load_count = int(params.get("load_count", 100))
+
+ try:
+ for i in range(load_count):
+ if not module_probe(mod_list):
+ raise error.TestFail("Failed to load module %s" % mod_list)
+ if not module_probe(mod_list, "-r"):
+ raise error.TestFail("Failed to remove module %s" % mod_list)
+ finally:
+ module_probe(mod_list)
diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample
index 1a260b4..8b2d752 100644
--- a/client/tests/kvm/tests_base.cfg.sample
+++ b/client/tests/kvm/tests_base.cfg.sample
@@ -748,6 +748,12 @@ variants:
- vmexit:
case = vmexit
+ - module_probe:
+ type = module_probe
+ # You can specify your own module list, though it is not needed usually.
+ # mod_list = kvm
+ load_count = 100
+
- ioquit:
type = ioquit
background_cmd = "for i in 1 2 3 4; do (nohup dd if=/dev/urandom of=/tmp/file bs=102400 count=10000000 &) done"
--
1.7.2.3
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 7/8] KVM test: add interface to get modules from cfg file, and load/unload them
2010-12-29 22:37 [PATCH 0/8] KVM test: Build code refactor + module_probe test Lucas Meneghel Rodrigues
` (5 preceding siblings ...)
2010-12-29 22:37 ` [PATCH 6/8] KVM test: load/unload kvm module Lucas Meneghel Rodrigues
@ 2010-12-29 22:37 ` Lucas Meneghel Rodrigues
2010-12-29 22:37 ` [PATCH 8/8] KVM test: test/module_probe: use installer object to load modules Lucas Meneghel Rodrigues
7 siblings, 0 replies; 9+ messages in thread
From: Lucas Meneghel Rodrigues @ 2010-12-29 22:37 UTC (permalink / raw)
To: autotest; +Cc: Suqin Huang, kvm
From: Suqin Huang <shuang@redhat.com>
The default modules are 'kvm, kvm_intel/amd, [ksm]'
user can set the module list in cfg file
the module names are splited with ","
Signed-off-by: Suqin Huang <shuang@redhat.com>
Acked-by: Chen Cao <kcao@redhat.com>
---
client/tests/kvm/tests/module_probe.py | 10 ++++++----
1 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/client/tests/kvm/tests/module_probe.py b/client/tests/kvm/tests/module_probe.py
index 519a916..727dcc2 100644
--- a/client/tests/kvm/tests/module_probe.py
+++ b/client/tests/kvm/tests/module_probe.py
@@ -6,7 +6,7 @@ def run_module_probe(test, params, env):
"""
load/unload kvm modules several times.
- Packet Loss Test:
+ Module load/unload Test:
1) check host cpu module
2) get module info
3) unload modules if they exist, else load them
@@ -35,11 +35,13 @@ def run_module_probe(test, params, env):
arch = "kvm_amd"
#Check whether ksm module exist
+ mod_str = ""
if os.path.exists("/sys/module/ksm"):
- mod_list = ["ksm", arch, "kvm"]
- else:
- mod_list = [arch, "kvm"]
+ mod_str = "ksm,"
+ mod_str += "%s, kvm" % arch
+ mod_str = params.get("mod_list", mod_str)
+ mod_list = re.split(",", mod_str)
logging.debug(mod_list)
load_count = int(params.get("load_count", 100))
--
1.7.2.3
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 8/8] KVM test: test/module_probe: use installer object to load modules
2010-12-29 22:37 [PATCH 0/8] KVM test: Build code refactor + module_probe test Lucas Meneghel Rodrigues
` (6 preceding siblings ...)
2010-12-29 22:37 ` [PATCH 7/8] KVM test: add interface to get modules from cfg file, and load/unload them Lucas Meneghel Rodrigues
@ 2010-12-29 22:37 ` Lucas Meneghel Rodrigues
7 siblings, 0 replies; 9+ messages in thread
From: Lucas Meneghel Rodrigues @ 2010-12-29 22:37 UTC (permalink / raw)
To: autotest; +Cc: Eduardo Habkost, kvm
From: Eduardo Habkost <ehabkost@redhat.com>
Instead of writing our own module loading code, use the load_modules()
method offerred by the installer object registered by the 'build' test.
If no previous installer was registered, assume the KVM modules were
pre-installed on the system, and use the PreInstalledKvm class.
For module unloading, use rmmod directly instead of using the
kvm.installer code because utils.unload_module() run too much stuff to
check if the module is available and figure out dependencies, and we
just want a simple 'rmmod' command to be run.
This removes the check for the 'ksm' module from the test code. If an
user wants the ksm module to be loaded/unload, it can be specified on
the 'extra_modules' parameter on the 'build' test, or on the
'module_probe' test itself, in case there's no 'build' test configured.
Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
---
client/tests/kvm/tests/module_probe.py | 81 ++++++++++++++++----------------
1 files changed, 41 insertions(+), 40 deletions(-)
diff --git a/client/tests/kvm/tests/module_probe.py b/client/tests/kvm/tests/module_probe.py
index 727dcc2..c370c4d 100644
--- a/client/tests/kvm/tests/module_probe.py
+++ b/client/tests/kvm/tests/module_probe.py
@@ -1,55 +1,56 @@
import re, commands, logging, os
-from autotest_lib.client.common_lib import error
-import kvm_subprocess, kvm_test_utils, kvm_utils
+from autotest_lib.client.common_lib import error, utils
+import kvm_subprocess, kvm_test_utils, kvm_utils, installer
+
def run_module_probe(test, params, env):
"""
- load/unload kvm modules several times.
+ load/unload KVM modules several times.
+
+ The test can run in two modes:
- Module load/unload Test:
- 1) check host cpu module
- 2) get module info
- 3) unload modules if they exist, else load them
+ - based on previous 'build' test: in case KVM modules were installed by a
+ 'build' test, we used the modules installed by the previous test.
- @param test: Kvm test object
- @param params: Dictionary with the test parameters
- @param env: Dictionary with test environment.
+ - based on own params: if no previous 'build' test was run,
+ we assume a pre-installed KVM module. Some parameters that
+ work for the 'build' can be used, then, such as 'extra_modules'.
"""
- def module_probe(name_list, arg=""):
- for name in name_list:
- cmd = "modprobe %s %s" % (arg, name)
- logging.debug(cmd)
- s, o = commands.getstatusoutput(cmd)
- if s != 0:
- logging.error("Failed to load/unload modules %s" % o)
- return False
- return True
-
- #Check host cpu module
- flags = file("/proc/cpuinfo").read()
- arch_check = re.findall("%s\s" % "vmx", flags)
- if arch_check:
- arch = "kvm_intel"
+ installer_object = env.previous_installer()
+ if installer_object is None:
+ installer_object = installer.PreInstalledKvm()
+ installer_object.set_install_params(test, params)
+
+ logging.debug('installer object: %r', installer_object)
+
+ mod_str = params.get("mod_list")
+ if mod_str:
+ mod_list = re.split("[, ]", mod_str)
+ logging.debug("mod list will be: %r", mod_list)
else:
- arch = "kvm_amd"
+ mod_list = installer_object.full_module_list()
+ logging.debug("mod list from installer: %r", mod_list)
- #Check whether ksm module exist
- mod_str = ""
- if os.path.exists("/sys/module/ksm"):
- mod_str = "ksm,"
- mod_str += "%s, kvm" % arch
+ # unload the modules before starting:
+ installer_object._unload_modules(mod_list)
- mod_str = params.get("mod_list", mod_str)
- mod_list = re.split(",", mod_str)
- logging.debug(mod_list)
load_count = int(params.get("load_count", 100))
-
try:
for i in range(load_count):
- if not module_probe(mod_list):
- raise error.TestFail("Failed to load module %s" % mod_list)
- if not module_probe(mod_list, "-r"):
- raise error.TestFail("Failed to remove module %s" % mod_list)
+ try:
+ installer_object.load_modules(mod_list)
+ except Exception,e:
+ raise error.TestFail("Failed to load modules [%r]: %s" %
+ (installer_object.full_module_list, e))
+
+ # unload using rmmod directly because utils.unload_module() (used by
+ # installer) does too much (runs lsmod, checks for dependencies),
+ # and we want to run the loop as fast as possible.
+ for mod in reversed(mod_list):
+ r = utils.system("rmmod %s" % (mod), ignore_status=True)
+ if r <> 0:
+ raise error.TestFail("Failed to unload module %s. "
+ "exit status: %d" % (mod, r))
finally:
- module_probe(mod_list)
+ installer_object.load_modules()
--
1.7.2.3
^ permalink raw reply related [flat|nested] 9+ messages in thread