* [PATCH] common_lib: Adding software_manager.py
@ 2010-11-29 2:44 Lucas Meneghel Rodrigues
2010-11-29 2:48 ` Lucas Meneghel Rodrigues
0 siblings, 1 reply; 2+ messages in thread
From: Lucas Meneghel Rodrigues @ 2010-11-29 2:44 UTC (permalink / raw)
To: autotest; +Cc: Ramon de Carvalho Valle, Higor Vieira Alves, kvm
This patch adds a software management abstraction layer
on top of popular distro package management abstraction
layers. Frequently we want to install a given distro
provided software package, but there's no single interface
to do so.
The software manager API tries to expose conveniently the
main package operations performed during software testing:
install() - Install a package or list of packages
remove() - Remove a package or list of packages
list_all() - List all packages installed
list_files() - List all files installed by a given package
add_repo() - Add a software repository
remove_repo() - Remove a software repository
So test writers don't have to care whether it is yum or
apt-get managing the system software. A rudimentary
command line interface for the API is provided, so
people can try out the methods.
The ideas of use inside autotest are:
* Replace all client.bin.package usage
* Usage with build_externals.py to install packages
* Provide a job.software_manager() attribute so people
can manipulate software install easily inside control
files and tests
The idea/very early draft came out more than 18 months
ago back in IBM (Ramon and Higor helped to write the
base methods), but only now I really sat to finish it
and sent it to the mailing list, to break the inertia :)
Tested/developed under the following systems:
* Fedora 13 (Yum backend)
* Ubuntu 10.04 (Apt backend)
* OpenSUSE 11.3 (Zypper backend)
Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
Signed-off-by: Higor Vieira Alves <halves@br.ibm.com>
Signed-off-by: Ramon de Carvalho Valle <rcvalle@br.ibm.com>
---
client/common_lib/software_manager.py | 788 +++++++++++++++++++++++++++++++++
1 files changed, 788 insertions(+), 0 deletions(-)
create mode 100755 client/common_lib/software_manager.py
diff --git a/client/common_lib/software_manager.py b/client/common_lib/software_manager.py
new file mode 100755
index 0000000..f67f667
--- /dev/null
+++ b/client/common_lib/software_manager.py
@@ -0,0 +1,788 @@
+#!/usr/bin/python
+"""
+Software package management library.
+
+This is an abstraction layer on top of the existing distributions high level
+package managers. It supports package operations useful for testing purposes,
+and multiple high level package managers (here called backends). If you want
+to make this lib to support your particular package manager/distro, please
+implement the given backend class.
+
+@author: Higor Vieira Alves (halves@br.ibm.com)
+@author: Lucas Meneghel Rodrigues (lmr@redhat.com)
+@author: Ramon de Carvalho Valle (rcvalle@br.ibm.com)
+
+@copyright: IBM 2008-2009
+@copyright: Red Hat 2009-2010
+"""
+import os, re, logging, ConfigParser, optparse, random, string
+try:
+ import yum
+except:
+ pass
+import common
+from autotest_lib.client.bin import os_dep, utils
+from autotest_lib.client.common_lib import error
+from autotest_lib.client.common_lib import logging_config, logging_manager
+
+
+def generate_random_string(length):
+ """
+ Return a random string using alphanumeric characters.
+
+ @length: Length of the string that will be generated.
+ """
+ r = random.SystemRandom()
+ str = ""
+ chars = string.letters + string.digits
+ while length > 0:
+ str += r.choice(chars)
+ length -= 1
+ return str
+
+
+class SoftwareManagerLoggingConfig(logging_config.LoggingConfig):
+ """
+ Used with the sole purpose of providing convenient logging setup
+ for the KVM test auxiliary programs.
+ """
+ def configure_logging(self, results_dir=None, verbose=False):
+ super(SoftwareManagerLoggingConfig, self).configure_logging(
+ use_console=True,
+ verbose=verbose)
+
+
+class SystemInspector(object):
+ """
+ System inspector class.
+
+ This may grow up to include more complete reports of operating system and
+ machine properties.
+ """
+ def __init__(self):
+ """
+ Probe system, and save information for future reference.
+ """
+ self.distro = utils.get_os_vendor()
+ self.high_level_pms = ['apt-get', 'yum', 'zypper']
+
+
+ def get_package_management(self):
+ """
+ Determine the supported package management systems present on the
+ system. If more than one package management system installed, try
+ to find the best supported system.
+ """
+ list_supported = []
+ for high_level_pm in self.high_level_pms:
+ try:
+ os_dep.command(high_level_pm)
+ list_supported.append(high_level_pm)
+ except:
+ pass
+
+ pm_supported = None
+ if len(list_supported) == 0:
+ pm_supported = None
+ if len(list_supported) == 1:
+ pm_supported = list_supported[0]
+ elif len(list_supported) > 1:
+ if 'apt-get' in list_supported and self.distro in ['Debian', 'Ubuntu']:
+ pm_supported = 'apt-get'
+ elif 'yum' in list_supported and self.distro == 'Fedora':
+ pm_supported = 'yum'
+ else:
+ pm_supported = list_supported[0]
+
+ logging.debug('Package Manager backend: %s' % pm_supported)
+ return pm_supported
+
+
+class SoftwareManager(object):
+ """
+ Package management abstraction layer.
+
+ It supports a set of common package operations for testing purposes, and it
+ uses the concept of a backend, a helper class that implements the set of
+ operations of a given package management tool.
+ """
+ def __init__(self):
+ """
+ Class constructor.
+
+ Determines the best supported package management system for the given
+ operating system running and initializes the appropriate backend.
+ """
+ inspector = SystemInspector()
+ backend_type = inspector.get_package_management()
+ if backend_type == 'yum':
+ self.backend = YumBackend()
+ elif backend_type == 'zypper':
+ self.backend = ZypperBackend()
+ elif backend_type == 'apt-get':
+ self.backend = AptBackend()
+ else:
+ raise NotImplementedError('Unimplemented package management '
+ 'system: %s.' % backend_type)
+
+
+ def check_installed(self, name, version=None, arch=None):
+ """
+ Check whether a package is installed on this system.
+
+ @param name: Package name.
+ @param version: Package version.
+ @param arch: Package architecture.
+ """
+ return self.backend.check_installed(name, version, arch)
+
+
+ def list_all(self):
+ """
+ List all installed packages.
+ """
+ return self.backend.list_all()
+
+
+ def list_files(self, name):
+ """
+ Get a list of all files installed by package [name].
+
+ @param name: Package name.
+ """
+ return self.backend.list_files(name)
+
+
+ def install(self, name):
+ """
+ Install package [name].
+
+ @param name: Package name.
+ """
+ return self.backend.install(name)
+
+
+ def remove(self, name):
+ """
+ Remove package [name].
+
+ @param name: Package name.
+ """
+ return self.backend.remove(name)
+
+
+ def add_repo(self, url):
+ """
+ Add package repo described by [url].
+
+ @param name: URL of the package repo.
+ """
+ return self.backend.add_repo(url)
+
+
+ def remove_repo(self, url):
+ """
+ Remove package repo described by [url].
+
+ @param url: URL of the package repo.
+ """
+ return self.backend.remove_repo(url)
+
+
+ def upgrade(self):
+ """
+ Upgrade all packages available.
+ """
+ return self.backend.upgrade()
+
+
+ def provides(self, file):
+ """
+ Returns a list of packages that provides a given capability to the
+ system (be it a binary, a library).
+
+ @param file: Path to the file.
+ """
+ return self.backend.provides(file)
+
+
+ def install_what_provides(self, file):
+ """
+ Installs package that provides [file].
+
+ @param file: Path to file.
+ """
+ provides = self.provides(file)
+ if provides is not None:
+ self.install(provides)
+ else:
+ logging.warning('No package seems to provide %s', file)
+
+
+class RpmBackend(object):
+ """
+ This class implements operations executed with the rpm package manager.
+
+ rpm is a lower level package manager, used by higher level managers such
+ as yum and zypper.
+ """
+ def __init__(self):
+ self.lowlevel_base_cmd = os_dep.command('rpm')
+
+
+ def _check_installed_version(self, name, version):
+ """
+ Helper for the check_installed public method.
+
+ @param name: Package name.
+ @param version: Package version.
+ """
+ cmd = (self.lowlevel_base_cmd + ' -q --qf %{VERSION} ' + name +
+ ' 2> /dev/null')
+ inst_version = utils.system_output(cmd)
+
+ if inst_version >= version:
+ return True
+ else:
+ return False
+
+
+ def check_installed(self, name, version=None, arch=None):
+ """
+ Check if package [name] is installed.
+
+ @param name: Package name.
+ @param version: Package version.
+ @param arch: Package architecture.
+ """
+ if arch:
+ cmd = (self.lowlevel_base_cmd + ' -q --qf %{ARCH} ' + name +
+ ' 2> /dev/null')
+ inst_archs = utils.system_output(cmd)
+ inst_archs = inst_archs.split('\n')
+
+ for inst_arch in inst_archs:
+ if inst_arch == arch:
+ return self._check_installed_version(name, version)
+ return False
+
+ elif version:
+ return self._check_installed_version(name, version)
+ else:
+ cmd = 'rpm -q ' + name + ' 2> /dev/null'
+ return (os.system(cmd) == 0)
+
+
+ def list_all(self):
+ """
+ List all installed packages.
+ """
+ installed_packages = utils.system_output('rpm -qa').splitlines()
+ return installed_packages
+
+
+ def list_files(self, name):
+ """
+ List files installed on the system by package [name].
+
+ @param name: Package name.
+ """
+ path = os.path.abspath(name)
+ if os.path.isfile(path):
+ option = '-qlp'
+ name = path
+ else:
+ option = '-ql'
+
+ l_cmd = 'rpm' + ' ' + option + ' ' + name + ' 2> /dev/null'
+
+ try:
+ result = utils.system_output(l_cmd)
+ list_files = result.split('\n')
+ return list_files
+ except error.CmdError:
+ return []
+
+
+class DpkgBackend(object):
+ """
+ This class implements operations executed with the dpkg package manager.
+
+ dpkg is a lower level package manager, used by higher level managers such
+ as apt and aptitude.
+ """
+ def __init__(self):
+ self.lowlevel_base_cmd = os_dep.command('dpkg')
+
+
+ def check_installed(self, name):
+ if os.path.isfile(name):
+ n_cmd = (self.lowlevel_base_cmd + ' -f ' + name +
+ ' Package 2>/dev/null')
+ name = utils.system_output(n_cmd)
+ i_cmd = self.lowlevel_base_cmd + ' -s ' + name + ' 2>/dev/null'
+ # Checking if package is installed
+ package_status = utils.system_output(i_cmd, ignore_status=True)
+ not_inst_pattern = re.compile('not-installed', re.IGNORECASE)
+ dpkg_not_installed = re.search(not_inst_pattern, package_status)
+ if dpkg_not_installed:
+ return False
+ return True
+
+
+ def list_all(self):
+ """
+ List all packages available in the system.
+ """
+ installed_packages = []
+ raw_list = utils.system_output('dpkg -l').splitlines()[5:]
+ for line in raw_list:
+ parts = line.split()
+ if parts[0] == "ii": # only grab "installed" packages
+ installed_packages.append("%s-%s" % (parts[1], parts[2]))
+
+
+ def list_files(self, package):
+ """
+ List files installed by package [package].
+
+ @param package: Package name.
+ @return: List of paths installed by package.
+ """
+ if os.path.isfile(package):
+ l_cmd = self.lowlevel_base_cmd + ' -c ' + package
+ else:
+ l_cmd = self.lowlevel_base_cmd + ' -l ' + package
+ return utils.system_output(l_cmd).split('\n')
+
+
+class YumBackend(RpmBackend):
+ """
+ Implements the yum backend for software manager.
+
+ Set of operations for the yum package manager, commonly found on Yellow Dog
+ Linux and Red Hat based distributions, such as Fedora and Red Hat
+ Enterprise Linux.
+ """
+ def __init__(self):
+ """
+ Initializes the base command and the yum package repository.
+ """
+ super(YumBackend, self).__init__()
+ executable = os_dep.command('yum')
+ base_arguments = '-y'
+ self.base_command = executable + ' ' + base_arguments
+ self.repo_file_path = '/etc/yum.repos.d/autotest.repo'
+ self.cfgparser = ConfigParser.ConfigParser()
+ self.cfgparser.read(self.repo_file_path)
+ y_cmd = executable + ' --version | head -1'
+ self.yum_version = utils.system_output(y_cmd, ignore_status=True)
+ logging.debug('Yum backend initialized')
+ logging.debug('Yum version: %s' % self.yum_version)
+ self.yum_base = yum.YumBase()
+
+
+ def _cleanup(self):
+ """
+ Clean up the yum cache so new package information can be downloaded.
+ """
+ utils.system("yum clean all")
+
+
+ def install(self, name):
+ """
+ Installs package [name]. Handles local installs.
+ """
+ if os.path.isfile(name):
+ name = os.path.abspath(name)
+ command = 'localinstall'
+ else:
+ command = 'install'
+
+ i_cmd = self.base_command + ' ' + command + ' ' + name
+
+ try:
+ utils.system(i_cmd)
+ return True
+ except:
+ return False
+
+
+ def remove(self, name):
+ """
+ Removes package [name].
+
+ @param name: Package name (eg. 'ipython').
+ """
+ r_cmd = self.base_command + ' ' + 'erase' + ' ' + name
+ try:
+ utils.system(r_cmd)
+ return True
+ except:
+ return False
+
+
+ def add_repo(self, url):
+ """
+ Adds package repository located on [url].
+
+ @param url: Universal Resource Locator of the repository.
+ """
+ # Check if we URL is already set
+ for section in self.cfgparser.sections():
+ for option, value in self.cfgparser.items(section):
+ if option == 'url' and value == url:
+ return True
+
+ # Didn't find it, let's set it up
+ while True:
+ section_name = 'software_manager' + '_' + generate_random_string(4)
+ if not self.cfgparser.has_section(section_name):
+ break
+ self.cfgparser.add_section(section_name)
+ self.cfgparser.set(section_name, 'name',
+ 'Repository added by the autotest software manager.')
+ self.cfgparser.set(section_name, 'url', url)
+ self.cfgparser.set(section_name, 'enabled', 1)
+ self.cfgparser.set(section_name, 'gpgcheck', 0)
+ self.cfgparser.write(self.repo_file_path)
+
+
+ def remove_repo(self, url):
+ """
+ Removes package repository located on [url].
+
+ @param url: Universal Resource Locator of the repository.
+ """
+ for section in self.cfgparser.sections():
+ for option, value in self.cfgparser.items(section):
+ if option == 'url' and value == url:
+ self.cfgparser.remove_section(section)
+ self.cfgparser.write(self.repo_file_path)
+
+
+ def upgrade(self):
+ """
+ Upgrade all available packages.
+ """
+ r_cmd = self.base_command + ' ' + 'update'
+ try:
+ utils.system(r_cmd)
+ return True
+ except:
+ return False
+
+
+ def provides(self, name):
+ """
+ Returns a list of packages that provides a given capability.
+
+ @param name: Capability name (eg, 'foo').
+ """
+ d_provides = self.yum_base.searchPackageProvides(args=[name])
+ provides_list = [key for key in d_provides]
+ if provides_list:
+ logging.info("Package %s provides %s", provides_list[0], name)
+ return str(provides_list[0])
+ else:
+ return None
+
+
+class ZypperBackend(RpmBackend):
+ """
+ Implements the zypper backend for software manager.
+
+ Set of operations for the zypper package manager, found on SUSE Linux.
+ """
+ def __init__(self):
+ """
+ Initializes the base command and the yum package repository.
+ """
+ super(ZypperBackend, self).__init__()
+ self.base_command = os_dep.command('zypper') + ' -n'
+ z_cmd = self.base_command + ' --version'
+ self.zypper_version = utils.system_output(z_cmd, ignore_status=True)
+ logging.debug('Zypper backend initialized')
+ logging.debug('Zypper version: %s' % self.zypper_version)
+
+
+ def install(self, name):
+ """
+ Installs package [name]. Handles local installs.
+
+ @param name: Package Name.
+ """
+ path = os.path.abspath(name)
+ i_cmd = self.base_command + ' install -l ' + name
+ try:
+ utils.system(i_cmd)
+ return True
+ except:
+ return False
+
+
+ def add_repo(self, url):
+ """
+ Adds repository [url].
+
+ @param url: URL for the package repository.
+ """
+ ar_cmd = self.base_command + ' addrepo ' + url
+ try:
+ utils.system(ar_cmd)
+ return True
+ except:
+ return False
+
+
+ def remove_repo(self, url):
+ """
+ Removes repository [url].
+
+ @param url: URL for the package repository.
+ """
+ rr_cmd = self.base_command + ' removerepo ' + url
+ try:
+ utils.system(rr_cmd)
+ return True
+ except:
+ return False
+
+
+ def remove(self, name):
+ """
+ Removes package [name].
+ """
+ r_cmd = self.base_command + ' ' + 'erase' + ' ' + name
+
+ try:
+ utils.system(r_cmd)
+ return True
+ except:
+ return False
+
+
+ def upgrade(self):
+ """
+ Upgrades all packages of the system.
+ """
+ u_cmd = self.base_command + ' update -l'
+
+ try:
+ utils.system(u_cmd)
+ return True
+ except:
+ return False
+
+
+ def provides(self, name):
+ """
+ Searches for what provides a given file.
+
+ @param name: File path.
+ """
+ p_cmd = self.base_command + ' what-provides ' + name
+ list_provides = []
+ try:
+ p_output = utils.system_output(p_cmd).split('\n')[4:]
+ for line in p_output:
+ line = [a.strip() for a in line.split('|')]
+ try:
+ state, pname, type, version, arch, repository = line
+ if pname not in list_provides:
+ list_provides.append(pname)
+ except IndexError:
+ pass
+ if len(list_provides) > 1:
+ logging.warning('More than one package found, '
+ 'opting by the first queue result')
+ if list_provides:
+ logging.info("Package %s provides %s", list_provides[0], name)
+ return list_provides[0]
+ return None
+ except:
+ return None
+
+
+class AptBackend(DpkgBackend):
+ """
+ Implements the apt backend for software manager.
+
+ Set of operations for the apt package manager, commonly found on Debian and
+ Debian based distributions, such as Ubuntu Linux.
+ """
+ def __init__(self):
+ """
+ Initializes the base command and the debian package repository.
+ """
+ super(AptBackend, self).__init__()
+ executable = os_dep.command('apt-get')
+ self.base_command = executable + ' -y'
+ self.repo_file_path = '/etc/apt/sources.list.d/autotest'
+ self.apt_version = utils.system_output('apt-get -v | head -1',
+ ignore_status=True)
+ logging.debug('Apt backend initialized')
+ logging.debug('apt version: %s' % self.apt_version)
+
+
+ def install(self, name):
+ """
+ Installs package [name].
+
+ @param name: Package name.
+ """
+ command = 'install'
+ i_cmd = self.base_command + ' ' + command + ' ' + name
+
+ try:
+ utils.system(i_cmd)
+ return True
+ except:
+ return False
+
+
+ def remove(self, name):
+ """
+ Remove package [name].
+
+ @param name: Package name.
+ """
+ command = 'remove'
+ flag = '--purge'
+ r_cmd = self.base_command + ' ' + command + ' ' + flag + ' ' + name
+
+ try:
+ utils.system(r_cmd)
+ return True
+ except:
+ return False
+
+
+ def add_repo(self, repo):
+ """
+ Add an apt repository.
+
+ @param repo: Repository string. Example:
+ 'deb http://archive.ubuntu.com/ubuntu/ maverick universe'
+ """
+ repo_file = open(self.repo_file_path, 'a')
+ repo_file_contents = repo_file.read()
+ if repo not in repo_file_contents:
+ repo_file.write(repo)
+
+
+ def remove_repo(self, repo):
+ """
+ Remove an apt repository.
+
+ @param repo: Repository string. Example:
+ 'deb http://archive.ubuntu.com/ubuntu/ maverick universe'
+ """
+ repo_file = open(self.repo_file_path, 'r')
+ new_file_contents = []
+ for line in repo_file.readlines:
+ if not line == repo:
+ new_file_contents.append(line)
+ repo_file.close()
+ new_file_contents = "\n".join(new_file_contents)
+ repo_file.open(self.repo_file_path, 'w')
+ repo_file.write(new_file_contents)
+ repo_file.close()
+
+
+ def upgrade(self):
+ """
+ Upgrade all packages of the system with eventual new versions.
+ """
+ ud_command = 'update'
+ ud_cmd = self.base_command + ' ' + ud_command
+ try:
+ utils.system(ud_cmd)
+ except:
+ logging.error("Apt package update failed")
+ up_command = 'upgrade'
+ up_cmd = self.base_command + ' ' + up_command
+ try:
+ utils.system(up_cmd)
+ return True
+ except:
+ return False
+
+
+ def provides(self, file):
+ """
+ Return a list of packages that provide [file].
+
+ @param file: File path.
+ """
+ if not self.check_installed('apt-file'):
+ self.install('apt-file')
+ command = os_dep.command('apt-file')
+ cache_update_cmd = command + ' update'
+ try:
+ utils.system(cache_update_cmd, ignore_status=True)
+ except:
+ logging.error("Apt file cache update failed")
+ fu_cmd = command + ' search ' + file
+ try:
+ provides = utils.system_output(fu_cmd).split('\n')
+ list_provides = []
+ for line in provides:
+ if line:
+ try:
+ line = line.split(':')
+ package = line[0].strip()
+ path = line[1].strip()
+ if path == file and package not in list_provides:
+ list_provides.append(package)
+ except IndexError:
+ pass
+ if len(list_provides) > 1:
+ logging.warning('More than one package found, '
+ 'opting by the first queue result')
+ if list_provides:
+ logging.info("Package %s provides %s", list_provides[0], file)
+ return list_provides[0]
+ return None
+ except:
+ return None
+
+
+if __name__ == '__main__':
+ parser = optparse.OptionParser(
+ "usage: %prog [install|remove|list-all|list-files|add-repo|remove-repo|"
+ "upgrade|what-provides|install-what-provides] arguments")
+ parser.add_option('--verbose', dest="debug", action='store_true',
+ help='include debug messages in console output')
+
+ options, args = parser.parse_args()
+ debug = options.debug
+ logging_manager.configure_logging(SoftwareManagerLoggingConfig(),
+ verbose=debug)
+ software_manager = SoftwareManager()
+ if args:
+ action = args[0]
+ args = " ".join(args[1:])
+ else:
+ action = 'show-help'
+
+ if action == 'install':
+ software_manager.install(args)
+ elif action == 'remove':
+ software_manager.remove(args)
+ if action == 'list-all':
+ software_manager.list_all()
+ elif action == 'list-files':
+ software_manager.list_files(args)
+ elif action == 'add-repo':
+ software_manager.add_repo(args)
+ elif action == 'remove-repo':
+ software_manager.remove_repo(args)
+ elif action == 'upgrade':
+ software_manager.upgrade()
+ elif action == 'what-provides':
+ software_manager.provides(args)
+ elif action == 'install-what-provides':
+ software_manager.install_what_provides(args)
+ elif action == 'show-help':
+ parser.print_help()
--
1.7.2.3
^ permalink raw reply related [flat|nested] 2+ messages in thread
* Re: [PATCH] common_lib: Adding software_manager.py
2010-11-29 2:44 [PATCH] common_lib: Adding software_manager.py Lucas Meneghel Rodrigues
@ 2010-11-29 2:48 ` Lucas Meneghel Rodrigues
0 siblings, 0 replies; 2+ messages in thread
From: Lucas Meneghel Rodrigues @ 2010-11-29 2:48 UTC (permalink / raw)
To: autotest; +Cc: kvm, Higor Vieira Alves, Ramon de Carvalho Valle
On Mon, 2010-11-29 at 00:44 -0200, Lucas Meneghel Rodrigues wrote:
> This patch adds a software management abstraction layer
> on top of popular distro package management abstraction
> layers. Frequently we want to install a given distro
> provided software package, but there's no single interface
> to do so.
>
> The software manager API tries to expose conveniently the
> main package operations performed during software testing:
>
> install() - Install a package or list of packages
> remove() - Remove a package or list of packages
> list_all() - List all packages installed
> list_files() - List all files installed by a given package
> add_repo() - Add a software repository
> remove_repo() - Remove a software repository
I forgot to mention
provides() - Return a list of packages that provide '/path/to/file'
install_what_provides() - Install package that provides '/path/to/file'
>
> So test writers don't have to care whether it is yum or
> apt-get managing the system software. A rudimentary
> command line interface for the API is provided, so
> people can try out the methods.
>
> The ideas of use inside autotest are:
> * Replace all client.bin.package usage
> * Usage with build_externals.py to install packages
> * Provide a job.software_manager() attribute so people
> can manipulate software install easily inside control
> files and tests
>
> The idea/very early draft came out more than 18 months
> ago back in IBM (Ramon and Higor helped to write the
> base methods), but only now I really sat to finish it
> and sent it to the mailing list, to break the inertia :)
>
> Tested/developed under the following systems:
> * Fedora 13 (Yum backend)
> * Ubuntu 10.04 (Apt backend)
> * OpenSUSE 11.3 (Zypper backend)
>
> Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
> Signed-off-by: Higor Vieira Alves <halves@br.ibm.com>
> Signed-off-by: Ramon de Carvalho Valle <rcvalle@br.ibm.com>
> ---
> client/common_lib/software_manager.py | 788 +++++++++++++++++++++++++++++++++
> 1 files changed, 788 insertions(+), 0 deletions(-)
> create mode 100755 client/common_lib/software_manager.py
>
> diff --git a/client/common_lib/software_manager.py b/client/common_lib/software_manager.py
> new file mode 100755
> index 0000000..f67f667
> --- /dev/null
> +++ b/client/common_lib/software_manager.py
> @@ -0,0 +1,788 @@
> +#!/usr/bin/python
> +"""
> +Software package management library.
> +
> +This is an abstraction layer on top of the existing distributions high level
> +package managers. It supports package operations useful for testing purposes,
> +and multiple high level package managers (here called backends). If you want
> +to make this lib to support your particular package manager/distro, please
> +implement the given backend class.
> +
> +@author: Higor Vieira Alves (halves@br.ibm.com)
> +@author: Lucas Meneghel Rodrigues (lmr@redhat.com)
> +@author: Ramon de Carvalho Valle (rcvalle@br.ibm.com)
> +
> +@copyright: IBM 2008-2009
> +@copyright: Red Hat 2009-2010
> +"""
> +import os, re, logging, ConfigParser, optparse, random, string
> +try:
> + import yum
> +except:
> + pass
> +import common
> +from autotest_lib.client.bin import os_dep, utils
> +from autotest_lib.client.common_lib import error
> +from autotest_lib.client.common_lib import logging_config, logging_manager
> +
> +
> +def generate_random_string(length):
> + """
> + Return a random string using alphanumeric characters.
> +
> + @length: Length of the string that will be generated.
> + """
> + r = random.SystemRandom()
> + str = ""
> + chars = string.letters + string.digits
> + while length > 0:
> + str += r.choice(chars)
> + length -= 1
> + return str
> +
> +
> +class SoftwareManagerLoggingConfig(logging_config.LoggingConfig):
> + """
> + Used with the sole purpose of providing convenient logging setup
> + for the KVM test auxiliary programs.
> + """
> + def configure_logging(self, results_dir=None, verbose=False):
> + super(SoftwareManagerLoggingConfig, self).configure_logging(
> + use_console=True,
> + verbose=verbose)
> +
> +
> +class SystemInspector(object):
> + """
> + System inspector class.
> +
> + This may grow up to include more complete reports of operating system and
> + machine properties.
> + """
> + def __init__(self):
> + """
> + Probe system, and save information for future reference.
> + """
> + self.distro = utils.get_os_vendor()
> + self.high_level_pms = ['apt-get', 'yum', 'zypper']
> +
> +
> + def get_package_management(self):
> + """
> + Determine the supported package management systems present on the
> + system. If more than one package management system installed, try
> + to find the best supported system.
> + """
> + list_supported = []
> + for high_level_pm in self.high_level_pms:
> + try:
> + os_dep.command(high_level_pm)
> + list_supported.append(high_level_pm)
> + except:
> + pass
> +
> + pm_supported = None
> + if len(list_supported) == 0:
> + pm_supported = None
> + if len(list_supported) == 1:
> + pm_supported = list_supported[0]
> + elif len(list_supported) > 1:
> + if 'apt-get' in list_supported and self.distro in ['Debian', 'Ubuntu']:
> + pm_supported = 'apt-get'
> + elif 'yum' in list_supported and self.distro == 'Fedora':
> + pm_supported = 'yum'
> + else:
> + pm_supported = list_supported[0]
> +
> + logging.debug('Package Manager backend: %s' % pm_supported)
> + return pm_supported
> +
> +
> +class SoftwareManager(object):
> + """
> + Package management abstraction layer.
> +
> + It supports a set of common package operations for testing purposes, and it
> + uses the concept of a backend, a helper class that implements the set of
> + operations of a given package management tool.
> + """
> + def __init__(self):
> + """
> + Class constructor.
> +
> + Determines the best supported package management system for the given
> + operating system running and initializes the appropriate backend.
> + """
> + inspector = SystemInspector()
> + backend_type = inspector.get_package_management()
> + if backend_type == 'yum':
> + self.backend = YumBackend()
> + elif backend_type == 'zypper':
> + self.backend = ZypperBackend()
> + elif backend_type == 'apt-get':
> + self.backend = AptBackend()
> + else:
> + raise NotImplementedError('Unimplemented package management '
> + 'system: %s.' % backend_type)
> +
> +
> + def check_installed(self, name, version=None, arch=None):
> + """
> + Check whether a package is installed on this system.
> +
> + @param name: Package name.
> + @param version: Package version.
> + @param arch: Package architecture.
> + """
> + return self.backend.check_installed(name, version, arch)
> +
> +
> + def list_all(self):
> + """
> + List all installed packages.
> + """
> + return self.backend.list_all()
> +
> +
> + def list_files(self, name):
> + """
> + Get a list of all files installed by package [name].
> +
> + @param name: Package name.
> + """
> + return self.backend.list_files(name)
> +
> +
> + def install(self, name):
> + """
> + Install package [name].
> +
> + @param name: Package name.
> + """
> + return self.backend.install(name)
> +
> +
> + def remove(self, name):
> + """
> + Remove package [name].
> +
> + @param name: Package name.
> + """
> + return self.backend.remove(name)
> +
> +
> + def add_repo(self, url):
> + """
> + Add package repo described by [url].
> +
> + @param name: URL of the package repo.
> + """
> + return self.backend.add_repo(url)
> +
> +
> + def remove_repo(self, url):
> + """
> + Remove package repo described by [url].
> +
> + @param url: URL of the package repo.
> + """
> + return self.backend.remove_repo(url)
> +
> +
> + def upgrade(self):
> + """
> + Upgrade all packages available.
> + """
> + return self.backend.upgrade()
> +
> +
> + def provides(self, file):
> + """
> + Returns a list of packages that provides a given capability to the
> + system (be it a binary, a library).
> +
> + @param file: Path to the file.
> + """
> + return self.backend.provides(file)
> +
> +
> + def install_what_provides(self, file):
> + """
> + Installs package that provides [file].
> +
> + @param file: Path to file.
> + """
> + provides = self.provides(file)
> + if provides is not None:
> + self.install(provides)
> + else:
> + logging.warning('No package seems to provide %s', file)
> +
> +
> +class RpmBackend(object):
> + """
> + This class implements operations executed with the rpm package manager.
> +
> + rpm is a lower level package manager, used by higher level managers such
> + as yum and zypper.
> + """
> + def __init__(self):
> + self.lowlevel_base_cmd = os_dep.command('rpm')
> +
> +
> + def _check_installed_version(self, name, version):
> + """
> + Helper for the check_installed public method.
> +
> + @param name: Package name.
> + @param version: Package version.
> + """
> + cmd = (self.lowlevel_base_cmd + ' -q --qf %{VERSION} ' + name +
> + ' 2> /dev/null')
> + inst_version = utils.system_output(cmd)
> +
> + if inst_version >= version:
> + return True
> + else:
> + return False
> +
> +
> + def check_installed(self, name, version=None, arch=None):
> + """
> + Check if package [name] is installed.
> +
> + @param name: Package name.
> + @param version: Package version.
> + @param arch: Package architecture.
> + """
> + if arch:
> + cmd = (self.lowlevel_base_cmd + ' -q --qf %{ARCH} ' + name +
> + ' 2> /dev/null')
> + inst_archs = utils.system_output(cmd)
> + inst_archs = inst_archs.split('\n')
> +
> + for inst_arch in inst_archs:
> + if inst_arch == arch:
> + return self._check_installed_version(name, version)
> + return False
> +
> + elif version:
> + return self._check_installed_version(name, version)
> + else:
> + cmd = 'rpm -q ' + name + ' 2> /dev/null'
> + return (os.system(cmd) == 0)
> +
> +
> + def list_all(self):
> + """
> + List all installed packages.
> + """
> + installed_packages = utils.system_output('rpm -qa').splitlines()
> + return installed_packages
> +
> +
> + def list_files(self, name):
> + """
> + List files installed on the system by package [name].
> +
> + @param name: Package name.
> + """
> + path = os.path.abspath(name)
> + if os.path.isfile(path):
> + option = '-qlp'
> + name = path
> + else:
> + option = '-ql'
> +
> + l_cmd = 'rpm' + ' ' + option + ' ' + name + ' 2> /dev/null'
> +
> + try:
> + result = utils.system_output(l_cmd)
> + list_files = result.split('\n')
> + return list_files
> + except error.CmdError:
> + return []
> +
> +
> +class DpkgBackend(object):
> + """
> + This class implements operations executed with the dpkg package manager.
> +
> + dpkg is a lower level package manager, used by higher level managers such
> + as apt and aptitude.
> + """
> + def __init__(self):
> + self.lowlevel_base_cmd = os_dep.command('dpkg')
> +
> +
> + def check_installed(self, name):
> + if os.path.isfile(name):
> + n_cmd = (self.lowlevel_base_cmd + ' -f ' + name +
> + ' Package 2>/dev/null')
> + name = utils.system_output(n_cmd)
> + i_cmd = self.lowlevel_base_cmd + ' -s ' + name + ' 2>/dev/null'
> + # Checking if package is installed
> + package_status = utils.system_output(i_cmd, ignore_status=True)
> + not_inst_pattern = re.compile('not-installed', re.IGNORECASE)
> + dpkg_not_installed = re.search(not_inst_pattern, package_status)
> + if dpkg_not_installed:
> + return False
> + return True
> +
> +
> + def list_all(self):
> + """
> + List all packages available in the system.
> + """
> + installed_packages = []
> + raw_list = utils.system_output('dpkg -l').splitlines()[5:]
> + for line in raw_list:
> + parts = line.split()
> + if parts[0] == "ii": # only grab "installed" packages
> + installed_packages.append("%s-%s" % (parts[1], parts[2]))
> +
> +
> + def list_files(self, package):
> + """
> + List files installed by package [package].
> +
> + @param package: Package name.
> + @return: List of paths installed by package.
> + """
> + if os.path.isfile(package):
> + l_cmd = self.lowlevel_base_cmd + ' -c ' + package
> + else:
> + l_cmd = self.lowlevel_base_cmd + ' -l ' + package
> + return utils.system_output(l_cmd).split('\n')
> +
> +
> +class YumBackend(RpmBackend):
> + """
> + Implements the yum backend for software manager.
> +
> + Set of operations for the yum package manager, commonly found on Yellow Dog
> + Linux and Red Hat based distributions, such as Fedora and Red Hat
> + Enterprise Linux.
> + """
> + def __init__(self):
> + """
> + Initializes the base command and the yum package repository.
> + """
> + super(YumBackend, self).__init__()
> + executable = os_dep.command('yum')
> + base_arguments = '-y'
> + self.base_command = executable + ' ' + base_arguments
> + self.repo_file_path = '/etc/yum.repos.d/autotest.repo'
> + self.cfgparser = ConfigParser.ConfigParser()
> + self.cfgparser.read(self.repo_file_path)
> + y_cmd = executable + ' --version | head -1'
> + self.yum_version = utils.system_output(y_cmd, ignore_status=True)
> + logging.debug('Yum backend initialized')
> + logging.debug('Yum version: %s' % self.yum_version)
> + self.yum_base = yum.YumBase()
> +
> +
> + def _cleanup(self):
> + """
> + Clean up the yum cache so new package information can be downloaded.
> + """
> + utils.system("yum clean all")
> +
> +
> + def install(self, name):
> + """
> + Installs package [name]. Handles local installs.
> + """
> + if os.path.isfile(name):
> + name = os.path.abspath(name)
> + command = 'localinstall'
> + else:
> + command = 'install'
> +
> + i_cmd = self.base_command + ' ' + command + ' ' + name
> +
> + try:
> + utils.system(i_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def remove(self, name):
> + """
> + Removes package [name].
> +
> + @param name: Package name (eg. 'ipython').
> + """
> + r_cmd = self.base_command + ' ' + 'erase' + ' ' + name
> + try:
> + utils.system(r_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def add_repo(self, url):
> + """
> + Adds package repository located on [url].
> +
> + @param url: Universal Resource Locator of the repository.
> + """
> + # Check if we URL is already set
> + for section in self.cfgparser.sections():
> + for option, value in self.cfgparser.items(section):
> + if option == 'url' and value == url:
> + return True
> +
> + # Didn't find it, let's set it up
> + while True:
> + section_name = 'software_manager' + '_' + generate_random_string(4)
> + if not self.cfgparser.has_section(section_name):
> + break
> + self.cfgparser.add_section(section_name)
> + self.cfgparser.set(section_name, 'name',
> + 'Repository added by the autotest software manager.')
> + self.cfgparser.set(section_name, 'url', url)
> + self.cfgparser.set(section_name, 'enabled', 1)
> + self.cfgparser.set(section_name, 'gpgcheck', 0)
> + self.cfgparser.write(self.repo_file_path)
> +
> +
> + def remove_repo(self, url):
> + """
> + Removes package repository located on [url].
> +
> + @param url: Universal Resource Locator of the repository.
> + """
> + for section in self.cfgparser.sections():
> + for option, value in self.cfgparser.items(section):
> + if option == 'url' and value == url:
> + self.cfgparser.remove_section(section)
> + self.cfgparser.write(self.repo_file_path)
> +
> +
> + def upgrade(self):
> + """
> + Upgrade all available packages.
> + """
> + r_cmd = self.base_command + ' ' + 'update'
> + try:
> + utils.system(r_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def provides(self, name):
> + """
> + Returns a list of packages that provides a given capability.
> +
> + @param name: Capability name (eg, 'foo').
> + """
> + d_provides = self.yum_base.searchPackageProvides(args=[name])
> + provides_list = [key for key in d_provides]
> + if provides_list:
> + logging.info("Package %s provides %s", provides_list[0], name)
> + return str(provides_list[0])
> + else:
> + return None
> +
> +
> +class ZypperBackend(RpmBackend):
> + """
> + Implements the zypper backend for software manager.
> +
> + Set of operations for the zypper package manager, found on SUSE Linux.
> + """
> + def __init__(self):
> + """
> + Initializes the base command and the yum package repository.
> + """
> + super(ZypperBackend, self).__init__()
> + self.base_command = os_dep.command('zypper') + ' -n'
> + z_cmd = self.base_command + ' --version'
> + self.zypper_version = utils.system_output(z_cmd, ignore_status=True)
> + logging.debug('Zypper backend initialized')
> + logging.debug('Zypper version: %s' % self.zypper_version)
> +
> +
> + def install(self, name):
> + """
> + Installs package [name]. Handles local installs.
> +
> + @param name: Package Name.
> + """
> + path = os.path.abspath(name)
> + i_cmd = self.base_command + ' install -l ' + name
> + try:
> + utils.system(i_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def add_repo(self, url):
> + """
> + Adds repository [url].
> +
> + @param url: URL for the package repository.
> + """
> + ar_cmd = self.base_command + ' addrepo ' + url
> + try:
> + utils.system(ar_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def remove_repo(self, url):
> + """
> + Removes repository [url].
> +
> + @param url: URL for the package repository.
> + """
> + rr_cmd = self.base_command + ' removerepo ' + url
> + try:
> + utils.system(rr_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def remove(self, name):
> + """
> + Removes package [name].
> + """
> + r_cmd = self.base_command + ' ' + 'erase' + ' ' + name
> +
> + try:
> + utils.system(r_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def upgrade(self):
> + """
> + Upgrades all packages of the system.
> + """
> + u_cmd = self.base_command + ' update -l'
> +
> + try:
> + utils.system(u_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def provides(self, name):
> + """
> + Searches for what provides a given file.
> +
> + @param name: File path.
> + """
> + p_cmd = self.base_command + ' what-provides ' + name
> + list_provides = []
> + try:
> + p_output = utils.system_output(p_cmd).split('\n')[4:]
> + for line in p_output:
> + line = [a.strip() for a in line.split('|')]
> + try:
> + state, pname, type, version, arch, repository = line
> + if pname not in list_provides:
> + list_provides.append(pname)
> + except IndexError:
> + pass
> + if len(list_provides) > 1:
> + logging.warning('More than one package found, '
> + 'opting by the first queue result')
> + if list_provides:
> + logging.info("Package %s provides %s", list_provides[0], name)
> + return list_provides[0]
> + return None
> + except:
> + return None
> +
> +
> +class AptBackend(DpkgBackend):
> + """
> + Implements the apt backend for software manager.
> +
> + Set of operations for the apt package manager, commonly found on Debian and
> + Debian based distributions, such as Ubuntu Linux.
> + """
> + def __init__(self):
> + """
> + Initializes the base command and the debian package repository.
> + """
> + super(AptBackend, self).__init__()
> + executable = os_dep.command('apt-get')
> + self.base_command = executable + ' -y'
> + self.repo_file_path = '/etc/apt/sources.list.d/autotest'
> + self.apt_version = utils.system_output('apt-get -v | head -1',
> + ignore_status=True)
> + logging.debug('Apt backend initialized')
> + logging.debug('apt version: %s' % self.apt_version)
> +
> +
> + def install(self, name):
> + """
> + Installs package [name].
> +
> + @param name: Package name.
> + """
> + command = 'install'
> + i_cmd = self.base_command + ' ' + command + ' ' + name
> +
> + try:
> + utils.system(i_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def remove(self, name):
> + """
> + Remove package [name].
> +
> + @param name: Package name.
> + """
> + command = 'remove'
> + flag = '--purge'
> + r_cmd = self.base_command + ' ' + command + ' ' + flag + ' ' + name
> +
> + try:
> + utils.system(r_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def add_repo(self, repo):
> + """
> + Add an apt repository.
> +
> + @param repo: Repository string. Example:
> + 'deb http://archive.ubuntu.com/ubuntu/ maverick universe'
> + """
> + repo_file = open(self.repo_file_path, 'a')
> + repo_file_contents = repo_file.read()
> + if repo not in repo_file_contents:
> + repo_file.write(repo)
> +
> +
> + def remove_repo(self, repo):
> + """
> + Remove an apt repository.
> +
> + @param repo: Repository string. Example:
> + 'deb http://archive.ubuntu.com/ubuntu/ maverick universe'
> + """
> + repo_file = open(self.repo_file_path, 'r')
> + new_file_contents = []
> + for line in repo_file.readlines:
> + if not line == repo:
> + new_file_contents.append(line)
> + repo_file.close()
> + new_file_contents = "\n".join(new_file_contents)
> + repo_file.open(self.repo_file_path, 'w')
> + repo_file.write(new_file_contents)
> + repo_file.close()
> +
> +
> + def upgrade(self):
> + """
> + Upgrade all packages of the system with eventual new versions.
> + """
> + ud_command = 'update'
> + ud_cmd = self.base_command + ' ' + ud_command
> + try:
> + utils.system(ud_cmd)
> + except:
> + logging.error("Apt package update failed")
> + up_command = 'upgrade'
> + up_cmd = self.base_command + ' ' + up_command
> + try:
> + utils.system(up_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def provides(self, file):
> + """
> + Return a list of packages that provide [file].
> +
> + @param file: File path.
> + """
> + if not self.check_installed('apt-file'):
> + self.install('apt-file')
> + command = os_dep.command('apt-file')
> + cache_update_cmd = command + ' update'
> + try:
> + utils.system(cache_update_cmd, ignore_status=True)
> + except:
> + logging.error("Apt file cache update failed")
> + fu_cmd = command + ' search ' + file
> + try:
> + provides = utils.system_output(fu_cmd).split('\n')
> + list_provides = []
> + for line in provides:
> + if line:
> + try:
> + line = line.split(':')
> + package = line[0].strip()
> + path = line[1].strip()
> + if path == file and package not in list_provides:
> + list_provides.append(package)
> + except IndexError:
> + pass
> + if len(list_provides) > 1:
> + logging.warning('More than one package found, '
> + 'opting by the first queue result')
> + if list_provides:
> + logging.info("Package %s provides %s", list_provides[0], file)
> + return list_provides[0]
> + return None
> + except:
> + return None
> +
> +
> +if __name__ == '__main__':
> + parser = optparse.OptionParser(
> + "usage: %prog [install|remove|list-all|list-files|add-repo|remove-repo|"
> + "upgrade|what-provides|install-what-provides] arguments")
> + parser.add_option('--verbose', dest="debug", action='store_true',
> + help='include debug messages in console output')
> +
> + options, args = parser.parse_args()
> + debug = options.debug
> + logging_manager.configure_logging(SoftwareManagerLoggingConfig(),
> + verbose=debug)
> + software_manager = SoftwareManager()
> + if args:
> + action = args[0]
> + args = " ".join(args[1:])
> + else:
> + action = 'show-help'
> +
> + if action == 'install':
> + software_manager.install(args)
> + elif action == 'remove':
> + software_manager.remove(args)
> + if action == 'list-all':
> + software_manager.list_all()
> + elif action == 'list-files':
> + software_manager.list_files(args)
> + elif action == 'add-repo':
> + software_manager.add_repo(args)
> + elif action == 'remove-repo':
> + software_manager.remove_repo(args)
> + elif action == 'upgrade':
> + software_manager.upgrade()
> + elif action == 'what-provides':
> + software_manager.provides(args)
> + elif action == 'install-what-provides':
> + software_manager.install_what_provides(args)
> + elif action == 'show-help':
> + parser.print_help()
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2010-11-29 2:48 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2010-11-29 2:44 [PATCH] common_lib: Adding software_manager.py Lucas Meneghel Rodrigues
2010-11-29 2:48 ` Lucas Meneghel Rodrigues
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox