From mboxrd@z Thu Jan 1 00:00:00 1970 From: Lucas Meneghel Rodrigues Subject: [PATCH 2/7] KVM test: Create autotest_lib.client.virt namespace Date: Wed, 9 Mar 2011 06:21:05 -0300 Message-ID: <1299662470-3017-3-git-send-email-lmr@redhat.com> References: <1299662470-3017-1-git-send-email-lmr@redhat.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Cc: kvm@vger.kernel.org To: autotest@test.kernel.org Return-path: In-Reply-To: <1299662470-3017-1-git-send-email-lmr@redhat.com> List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Sender: autotest-bounces@test.kernel.org Errors-To: autotest-bounces@test.kernel.org List-Id: kvm.vger.kernel.org This patch moves all libraries from the KVM test to autotest_lib.client.virt, and makes the needed adaptations and abstracts the KVM implementation from the generic infrastructure. Signed-off-by: Lucas Meneghel Rodrigues --- client/common_lib/cartesian_config.py | 698 +++++++++++++ client/virt/aexpect.py | 1352 +++++++++++++++++++++++++ client/virt/kvm_installer.py | 797 +++++++++++++++ client/virt/kvm_monitor.py | 745 ++++++++++++++ client/virt/kvm_vm.py | 1500 ++++++++++++++++++++++++++++ client/virt/ppm_utils.py | 237 +++++ client/virt/rss_client.py | 519 ++++++++++ client/virt/virt_env_process.py | 438 ++++++++ client/virt/virt_scheduler.py | 229 +++++ client/virt/virt_step_editor.py | 1401 ++++++++++++++++++++++++++ client/virt/virt_test_setup.py | 700 +++++++++++++ client/virt/virt_test_utils.py | 754 ++++++++++++++ client/virt/virt_utils.py | 1760 +++++++++++++++++++++++++++++++++ client/virt/virt_vm.py | 298 ++++++ 14 files changed, 11428 insertions(+), 0 deletions(-) create mode 100755 client/common_lib/cartesian_config.py create mode 100644 client/virt/__init__.py create mode 100755 client/virt/aexpect.py create mode 100644 client/virt/kvm_installer.py create mode 100644 client/virt/kvm_monitor.py create mode 100755 client/virt/kvm_vm.py create mode 100644 client/virt/ppm_utils.py create mode 100755 client/virt/rss_client.py create mode 100644 client/virt/virt_env_process.py create mode 100644 client/virt/virt_scheduler.py create mode 100755 client/virt/virt_step_editor.py create mode 100644 client/virt/virt_test_setup.py create mode 100644 client/virt/virt_test_utils.py create mode 100644 client/virt/virt_utils.py create mode 100644 client/virt/virt_vm.py diff --git a/client/common_lib/cartesian_config.py b/client/common_lib/cartesian_config.py new file mode 100755 index 0000000..daf45d1 --- /dev/null +++ b/client/common_lib/cartesian_config.py @@ -0,0 +1,698 @@ +#!/usr/bin/python +""" +Cartesian configuration format file parser + +@copyright: Red Hat 2008-2011 +""" + +import re, os, sys, optparse, collections + + +# Filter syntax: +# , means OR +# .. means AND +# . means IMMEDIATELY-FOLLOWED-BY + +# Example: +# qcow2..Fedora.14, RHEL.6..raw..boot, smp2..qcow2..migrate..ide +# means match all dicts whose names have: +# (qcow2 AND (Fedora IMMEDIATELY-FOLLOWED-BY 14)) OR +# ((RHEL IMMEDIATELY-FOLLOWED-BY 6) AND raw AND boot) OR +# (smp2 AND qcow2 AND migrate AND ide) + +# Note: +# 'qcow2..Fedora.14' is equivalent to 'Fedora.14..qcow2'. +# 'qcow2..Fedora.14' is not equivalent to 'qcow2..14.Fedora'. +# 'ide, scsi' is equivalent to 'scsi, ide'. + +# Filters can be used in 3 ways: +# only +# no +# : +# The last one starts a conditional block. + + +class ParserError: + def __init__(self, msg, line=None, filename=None, linenum=None): + self.msg = msg + self.line = line + self.filename = filename + self.linenum = linenum + + def __str__(self): + if self.line: + return "%s: %r (%s:%s)" % (self.msg, self.line, + self.filename, self.linenum) + else: + return "%s (%s:%s)" % (self.msg, self.filename, self.linenum) + + +num_failed_cases = 5 + + +class Node(object): + def __init__(self): + self.name = [] + self.dep = [] + self.content = [] + self.children = [] + self.labels = set() + self.append_to_shortname = False + self.failed_cases = collections.deque() + + +def _match_adjacent(block, ctx, ctx_set): + # TODO: explain what this function does + if block[0] not in ctx_set: + return 0 + if len(block) == 1: + return 1 + if block[1] not in ctx_set: + return int(ctx[-1] == block[0]) + k = 0 + i = ctx.index(block[0]) + while i < len(ctx): + if k > 0 and ctx[i] != block[k]: + i -= k - 1 + k = 0 + if ctx[i] == block[k]: + k += 1 + if k >= len(block): + break + if block[k] not in ctx_set: + break + i += 1 + return k + + +def _might_match_adjacent(block, ctx, ctx_set, descendant_labels): + matched = _match_adjacent(block, ctx, ctx_set) + for elem in block[matched:]: + if elem not in descendant_labels: + return False + return True + + +# Filter must inherit from object (otherwise type() won't work) +class Filter(object): + def __init__(self, s): + self.filter = [] + for char in s: + if not (char.isalnum() or char.isspace() or char in ".,_-"): + raise ParserError("Illegal characters in filter") + for word in s.replace(",", " ").split(): + word = [block.split(".") for block in word.split("..")] + for block in word: + for elem in block: + if not elem: + raise ParserError("Syntax error") + self.filter += [word] + + + def match(self, ctx, ctx_set): + for word in self.filter: + for block in word: + if _match_adjacent(block, ctx, ctx_set) != len(block): + break + else: + return True + return False + + + def might_match(self, ctx, ctx_set, descendant_labels): + for word in self.filter: + for block in word: + if not _might_match_adjacent(block, ctx, ctx_set, + descendant_labels): + break + else: + return True + return False + + +class NoOnlyFilter(Filter): + def __init__(self, line): + Filter.__init__(self, line.split(None, 1)[1]) + self.line = line + + +class OnlyFilter(NoOnlyFilter): + def is_irrelevant(self, ctx, ctx_set, descendant_labels): + return self.match(ctx, ctx_set) + + + def requires_action(self, ctx, ctx_set, descendant_labels): + return not self.might_match(ctx, ctx_set, descendant_labels) + + + def might_pass(self, failed_ctx, failed_ctx_set, ctx, ctx_set, + descendant_labels): + for word in self.filter: + for block in word: + if (_match_adjacent(block, ctx, ctx_set) > + _match_adjacent(block, failed_ctx, failed_ctx_set)): + return self.might_match(ctx, ctx_set, descendant_labels) + return False + + +class NoFilter(NoOnlyFilter): + def is_irrelevant(self, ctx, ctx_set, descendant_labels): + return not self.might_match(ctx, ctx_set, descendant_labels) + + + def requires_action(self, ctx, ctx_set, descendant_labels): + return self.match(ctx, ctx_set) + + + def might_pass(self, failed_ctx, failed_ctx_set, ctx, ctx_set, + descendant_labels): + for word in self.filter: + for block in word: + if (_match_adjacent(block, ctx, ctx_set) < + _match_adjacent(block, failed_ctx, failed_ctx_set)): + return not self.match(ctx, ctx_set) + return False + + +class Condition(NoFilter): + def __init__(self, line): + Filter.__init__(self, line.rstrip(":")) + self.line = line + self.content = [] + + +class NegativeCondition(OnlyFilter): + def __init__(self, line): + Filter.__init__(self, line.lstrip("!").rstrip(":")) + self.line = line + self.content = [] + + +class Parser(object): + """ + Parse an input file or string that follows the KVM Test Config File format + and generate a list of dicts that will be later used as configuration + parameters by the KVM tests. + + @see: http://www.linux-kvm.org/page/KVM-Autotest/Test_Config_File + """ + + def __init__(self, filename=None, debug=False): + """ + Initialize the parser and optionally parse a file. + + @param filename: Path of the file to parse. + @param debug: Whether to turn on debugging output. + """ + self.node = Node() + self.debug = debug + if filename: + self.parse_file(filename) + + + def parse_file(self, filename): + """ + Parse a file. + + @param filename: Path of the configuration file. + """ + self.node = self._parse(FileReader(filename), self.node) + + + def parse_string(self, s): + """ + Parse a string. + + @param s: String to parse. + """ + self.node = self._parse(StrReader(s), self.node) + + + def get_dicts(self, node=None, ctx=[], content=[], shortname=[], dep=[]): + """ + Generate dictionaries from the code parsed so far. This should + be called after parsing something. + + @return: A dict generator. + """ + def process_content(content, failed_filters): + # 1. Check that the filters in content are OK with the current + # context (ctx). + # 2. Move the parts of content that are still relevant into + # new_content and unpack conditional blocks if appropriate. + # For example, if an 'only' statement fully matches ctx, it + # becomes irrelevant and is not appended to new_content. + # If a conditional block fully matches, its contents are + # unpacked into new_content. + # 3. Move failed filters into failed_filters, so that next time we + # reach this node or one of its ancestors, we'll check those + # filters first. + for t in content: + filename, linenum, obj = t + if type(obj) is Op: + new_content.append(t) + continue + # obj is an OnlyFilter/NoFilter/Condition/NegativeCondition + if obj.requires_action(ctx, ctx_set, labels): + # This filter requires action now + if type(obj) is OnlyFilter or type(obj) is NoFilter: + self._debug(" filter did not pass: %r (%s:%s)", + obj.line, filename, linenum) + failed_filters.append(t) + return False + else: + self._debug(" conditional block matches: %r (%s:%s)", + obj.line, filename, linenum) + # Check and unpack the content inside this Condition + # object (note: the failed filters should go into + # new_internal_filters because we don't expect them to + # come from outside this node, even if the Condition + # itself was external) + if not process_content(obj.content, + new_internal_filters): + failed_filters.append(t) + return False + continue + elif obj.is_irrelevant(ctx, ctx_set, labels): + # This filter is no longer relevant and can be removed + continue + else: + # Keep the filter and check it again later + new_content.append(t) + return True + + def might_pass(failed_ctx, + failed_ctx_set, + failed_external_filters, + failed_internal_filters): + for t in failed_external_filters: + if t not in content: + return True + filename, linenum, filter = t + if filter.might_pass(failed_ctx, failed_ctx_set, ctx, ctx_set, + labels): + return True + for t in failed_internal_filters: + filename, linenum, filter = t + if filter.might_pass(failed_ctx, failed_ctx_set, ctx, ctx_set, + labels): + return True + return False + + def add_failed_case(): + node.failed_cases.appendleft((ctx, ctx_set, + new_external_filters, + new_internal_filters)) + if len(node.failed_cases) > num_failed_cases: + node.failed_cases.pop() + + node = node or self.node + # Update dep + for d in node.dep: + dep = dep + [".".join(ctx + [d])] + # Update ctx + ctx = ctx + node.name + ctx_set = set(ctx) + labels = node.labels + # Get the current name + name = ".".join(ctx) + if node.name: + self._debug("checking out %r", name) + # Check previously failed filters + for i, failed_case in enumerate(node.failed_cases): + if not might_pass(*failed_case): + self._debug(" this subtree has failed before") + del node.failed_cases[i] + node.failed_cases.appendleft(failed_case) + return + # Check content and unpack it into new_content + new_content = [] + new_external_filters = [] + new_internal_filters = [] + if (not process_content(node.content, new_internal_filters) or + not process_content(content, new_external_filters)): + add_failed_case() + return + # Update shortname + if node.append_to_shortname: + shortname = shortname + node.name + # Recurse into children + count = 0 + for n in node.children: + for d in self.get_dicts(n, ctx, new_content, shortname, dep): + count += 1 + yield d + # Reached leaf? + if not node.children: + self._debug(" reached leaf, returning it") + d = {"name": name, "dep": dep, "shortname": ".".join(shortname)} + for filename, linenum, op in new_content: + op.apply_to_dict(d) + yield d + # If this node did not produce any dicts, remember the failed filters + # of its descendants + elif not count: + new_external_filters = [] + new_internal_filters = [] + for n in node.children: + (failed_ctx, + failed_ctx_set, + failed_external_filters, + failed_internal_filters) = n.failed_cases[0] + for obj in failed_internal_filters: + if obj not in new_internal_filters: + new_internal_filters.append(obj) + for obj in failed_external_filters: + if obj in content: + if obj not in new_external_filters: + new_external_filters.append(obj) + else: + if obj not in new_internal_filters: + new_internal_filters.append(obj) + add_failed_case() + + + def _debug(self, s, *args): + if self.debug: + s = "DEBUG: %s" % s + print s % args + + + def _warn(self, s, *args): + s = "WARNING: %s" % s + print s % args + + + def _parse_variants(self, cr, node, prev_indent=-1): + """ + Read and parse lines from a FileReader object until a line with an + indent level lower than or equal to prev_indent is encountered. + + @param cr: A FileReader/StrReader object. + @param node: A node to operate on. + @param prev_indent: The indent level of the "parent" block. + @return: A node object. + """ + node4 = Node() + + while True: + line, indent, linenum = cr.get_next_line(prev_indent) + if not line: + break + + name, dep = map(str.strip, line.lstrip("- ").split(":", 1)) + for char in name: + if not (char.isalnum() or char in "@._-"): + raise ParserError("Illegal characters in variant name", + line, cr.filename, linenum) + for char in dep: + if not (char.isalnum() or char.isspace() or char in ".,_-"): + raise ParserError("Illegal characters in dependencies", + line, cr.filename, linenum) + + node2 = Node() + node2.children = [node] + node2.labels = node.labels + + node3 = self._parse(cr, node2, prev_indent=indent) + node3.name = name.lstrip("@").split(".") + node3.dep = dep.replace(",", " ").split() + node3.append_to_shortname = not name.startswith("@") + + node4.children += [node3] + node4.labels.update(node3.labels) + node4.labels.update(node3.name) + + return node4 + + + def _parse(self, cr, node, prev_indent=-1): + """ + Read and parse lines from a StrReader object until a line with an + indent level lower than or equal to prev_indent is encountered. + + @param cr: A FileReader/StrReader object. + @param node: A Node or a Condition object to operate on. + @param prev_indent: The indent level of the "parent" block. + @return: A node object. + """ + while True: + line, indent, linenum = cr.get_next_line(prev_indent) + if not line: + break + + words = line.split(None, 1) + + # Parse 'variants' + if line == "variants:": + # 'variants' is not allowed inside a conditional block + if (isinstance(node, Condition) or + isinstance(node, NegativeCondition)): + raise ParserError("'variants' is not allowed inside a " + "conditional block", + None, cr.filename, linenum) + node = self._parse_variants(cr, node, prev_indent=indent) + continue + + # Parse 'include' statements + if words[0] == "include": + if len(words) < 2: + raise ParserError("Syntax error: missing parameter", + line, cr.filename, linenum) + filename = os.path.expanduser(words[1]) + if isinstance(cr, FileReader) and not os.path.isabs(filename): + filename = os.path.join(os.path.dirname(cr.filename), + filename) + if not os.path.isfile(filename): + self._warn("%r (%s:%s): file doesn't exist or is not a " + "regular file", line, cr.filename, linenum) + continue + node = self._parse(FileReader(filename), node) + continue + + # Parse 'only' and 'no' filters + if words[0] in ("only", "no"): + if len(words) < 2: + raise ParserError("Syntax error: missing parameter", + line, cr.filename, linenum) + try: + if words[0] == "only": + f = OnlyFilter(line) + elif words[0] == "no": + f = NoFilter(line) + except ParserError, e: + e.line = line + e.filename = cr.filename + e.linenum = linenum + raise + node.content += [(cr.filename, linenum, f)] + continue + + # Look for operators + op_match = _ops_exp.search(line) + + # Parse conditional blocks + if ":" in line: + index = line.index(":") + if not op_match or index < op_match.start(): + index += 1 + cr.set_next_line(line[index:], indent, linenum) + line = line[:index] + try: + if line.startswith("!"): + cond = NegativeCondition(line) + else: + cond = Condition(line) + except ParserError, e: + e.line = line + e.filename = cr.filename + e.linenum = linenum + raise + self._parse(cr, cond, prev_indent=indent) + node.content += [(cr.filename, linenum, cond)] + continue + + # Parse regular operators + if not op_match: + raise ParserError("Syntax error", line, cr.filename, linenum) + node.content += [(cr.filename, linenum, Op(line, op_match))] + + return node + + +# Assignment operators + +_reserved_keys = set(("name", "shortname", "dep")) + + +def _op_set(d, key, value): + if key not in _reserved_keys: + d[key] = value + + +def _op_append(d, key, value): + if key not in _reserved_keys: + d[key] = d.get(key, "") + value + + +def _op_prepend(d, key, value): + if key not in _reserved_keys: + d[key] = value + d.get(key, "") + + +def _op_regex_set(d, exp, value): + exp = re.compile("%s$" % exp) + for key in d: + if key not in _reserved_keys and exp.match(key): + d[key] = value + + +def _op_regex_append(d, exp, value): + exp = re.compile("%s$" % exp) + for key in d: + if key not in _reserved_keys and exp.match(key): + d[key] += value + + +def _op_regex_prepend(d, exp, value): + exp = re.compile("%s$" % exp) + for key in d: + if key not in _reserved_keys and exp.match(key): + d[key] = value + d[key] + + +def _op_regex_del(d, empty, exp): + exp = re.compile("%s$" % exp) + for key in d.keys(): + if key not in _reserved_keys and exp.match(key): + del d[key] + + +_ops = {"=": (r"\=", _op_set), + "+=": (r"\+\=", _op_append), + "<=": (r"\<\=", _op_prepend), + "?=": (r"\?\=", _op_regex_set), + "?+=": (r"\?\+\=", _op_regex_append), + "?<=": (r"\?\<\=", _op_regex_prepend), + "del": (r"^del\b", _op_regex_del)} + +_ops_exp = re.compile("|".join([op[0] for op in _ops.values()])) + + +class Op(object): + def __init__(self, line, m): + self.func = _ops[m.group()][1] + self.key = line[:m.start()].strip() + value = line[m.end():].strip() + if value and (value[0] == value[-1] == '"' or + value[0] == value[-1] == "'"): + value = value[1:-1] + self.value = value + + + def apply_to_dict(self, d): + self.func(d, self.key, self.value) + + +# StrReader and FileReader + +class StrReader(object): + """ + Preprocess an input string for easy reading. + """ + def __init__(self, s): + """ + Initialize the reader. + + @param s: The string to parse. + """ + self.filename = "" + self._lines = [] + self._line_index = 0 + self._stored_line = None + for linenum, line in enumerate(s.splitlines()): + line = line.rstrip().expandtabs() + stripped_line = line.lstrip() + indent = len(line) - len(stripped_line) + if (not stripped_line + or stripped_line.startswith("#") + or stripped_line.startswith("//")): + continue + self._lines.append((stripped_line, indent, linenum + 1)) + + + def get_next_line(self, prev_indent): + """ + Get the next line in the current block. + + @param prev_indent: The indentation level of the previous block. + @return: (line, indent, linenum), where indent is the line's + indentation level. If no line is available, (None, -1, -1) is + returned. + """ + if self._stored_line: + ret = self._stored_line + self._stored_line = None + return ret + if self._line_index >= len(self._lines): + return None, -1, -1 + line, indent, linenum = self._lines[self._line_index] + if indent <= prev_indent: + return None, -1, -1 + self._line_index += 1 + return line, indent, linenum + + + def set_next_line(self, line, indent, linenum): + """ + Make the next call to get_next_line() return the given line instead of + the real next line. + """ + line = line.strip() + if line: + self._stored_line = line, indent, linenum + + +class FileReader(StrReader): + """ + Preprocess an input file for easy reading. + """ + def __init__(self, filename): + """ + Initialize the reader. + + @parse filename: The name of the input file. + """ + StrReader.__init__(self, open(filename).read()) + self.filename = filename + + +if __name__ == "__main__": + parser = optparse.OptionParser('usage: %prog [options] filename ' + '[extra code] ...\n\nExample:\n\n ' + '%prog tests.cfg "only my_set" "no qcow2"') + parser.add_option("-v", "--verbose", dest="debug", action="store_true", + help="include debug messages in console output") + parser.add_option("-f", "--fullname", dest="fullname", action="store_true", + help="show full dict names instead of short names") + parser.add_option("-c", "--contents", dest="contents", action="store_true", + help="show dict contents") + + options, args = parser.parse_args() + if not args: + parser.error("filename required") + + c = Parser(args[0], debug=options.debug) + for s in args[1:]: + c.parse_string(s) + + for i, d in enumerate(c.get_dicts()): + if options.fullname: + print "dict %4d: %s" % (i + 1, d["name"]) + else: + print "dict %4d: %s" % (i + 1, d["shortname"]) + if options.contents: + keys = d.keys() + keys.sort() + for key in keys: + print " %s = %s" % (key, d[key]) diff --git a/client/virt/__init__.py b/client/virt/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/client/virt/aexpect.py b/client/virt/aexpect.py new file mode 100755 index 0000000..fe3f9ec --- /dev/null +++ b/client/virt/aexpect.py @@ -0,0 +1,1352 @@ +#!/usr/bin/python +""" +A class and functions used for running and controlling child processes. + +@copyright: 2008-2009 Red Hat Inc. +""" + +import os, sys, pty, select, termios, fcntl + + +# The following helper functions are shared by the server and the client. + +def _lock(filename): + if not os.path.exists(filename): + open(filename, "w").close() + fd = os.open(filename, os.O_RDWR) + fcntl.lockf(fd, fcntl.LOCK_EX) + return fd + + +def _unlock(fd): + fcntl.lockf(fd, fcntl.LOCK_UN) + os.close(fd) + + +def _locked(filename): + try: + fd = os.open(filename, os.O_RDWR) + except: + return False + try: + fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except: + os.close(fd) + return True + fcntl.lockf(fd, fcntl.LOCK_UN) + os.close(fd) + return False + + +def _wait(filename): + fd = _lock(filename) + _unlock(fd) + + +def _get_filenames(base_dir, id): + return [os.path.join(base_dir, s + id) for s in + "shell-pid-", "status-", "output-", "inpipe-", + "lock-server-running-", "lock-client-starting-"] + + +def _get_reader_filename(base_dir, id, reader): + return os.path.join(base_dir, "outpipe-%s-%s" % (reader, id)) + + +# The following is the server part of the module. + +if __name__ == "__main__": + id = sys.stdin.readline().strip() + echo = sys.stdin.readline().strip() == "True" + readers = sys.stdin.readline().strip().split(",") + command = sys.stdin.readline().strip() + " && echo %s > /dev/null" % id + + # Define filenames to be used for communication + base_dir = "/tmp/kvm_spawn" + (shell_pid_filename, + status_filename, + output_filename, + inpipe_filename, + lock_server_running_filename, + lock_client_starting_filename) = _get_filenames(base_dir, id) + + # Populate the reader filenames list + reader_filenames = [_get_reader_filename(base_dir, id, reader) + for reader in readers] + + # Set $TERM = dumb + os.putenv("TERM", "dumb") + + (shell_pid, shell_fd) = pty.fork() + if shell_pid == 0: + # Child process: run the command in a subshell + os.execv("/bin/sh", ["/bin/sh", "-c", command]) + else: + # Parent process + lock_server_running = _lock(lock_server_running_filename) + + # Set terminal echo on/off and disable pre- and post-processing + attr = termios.tcgetattr(shell_fd) + attr[0] &= ~termios.INLCR + attr[0] &= ~termios.ICRNL + attr[0] &= ~termios.IGNCR + attr[1] &= ~termios.OPOST + if echo: + attr[3] |= termios.ECHO + else: + attr[3] &= ~termios.ECHO + termios.tcsetattr(shell_fd, termios.TCSANOW, attr) + + # Open output file + output_file = open(output_filename, "w") + # Open input pipe + os.mkfifo(inpipe_filename) + inpipe_fd = os.open(inpipe_filename, os.O_RDWR) + # Open output pipes (readers) + reader_fds = [] + for filename in reader_filenames: + os.mkfifo(filename) + reader_fds.append(os.open(filename, os.O_RDWR)) + + # Write shell PID to file + file = open(shell_pid_filename, "w") + file.write(str(shell_pid)) + file.close() + + # Print something to stdout so the client can start working + print "Server %s ready" % id + sys.stdout.flush() + + # Initialize buffers + buffers = ["" for reader in readers] + + # Read from child and write to files/pipes + while True: + check_termination = False + # Make a list of reader pipes whose buffers are not empty + fds = [fd for (i, fd) in enumerate(reader_fds) if buffers[i]] + # Wait until there's something to do + r, w, x = select.select([shell_fd, inpipe_fd], fds, [], 0.5) + # If a reader pipe is ready for writing -- + for (i, fd) in enumerate(reader_fds): + if fd in w: + bytes_written = os.write(fd, buffers[i]) + buffers[i] = buffers[i][bytes_written:] + # If there's data to read from the child process -- + if shell_fd in r: + try: + data = os.read(shell_fd, 16384) + except OSError: + data = "" + if not data: + check_termination = True + # Remove carriage returns from the data -- they often cause + # trouble and are normally not needed + data = data.replace("\r", "") + output_file.write(data) + output_file.flush() + for i in range(len(readers)): + buffers[i] += data + # If os.read() raised an exception or there was nothing to read -- + if check_termination or shell_fd not in r: + pid, status = os.waitpid(shell_pid, os.WNOHANG) + if pid: + status = os.WEXITSTATUS(status) + break + # If there's data to read from the client -- + if inpipe_fd in r: + data = os.read(inpipe_fd, 1024) + os.write(shell_fd, data) + + # Write the exit status to a file + file = open(status_filename, "w") + file.write(str(status)) + file.close() + + # Wait for the client to finish initializing + _wait(lock_client_starting_filename) + + # Delete FIFOs + for filename in reader_filenames + [inpipe_filename]: + try: + os.unlink(filename) + except OSError: + pass + + # Close all files and pipes + output_file.close() + os.close(inpipe_fd) + for fd in reader_fds: + os.close(fd) + + _unlock(lock_server_running) + exit(0) + + +# The following is the client part of the module. + +import subprocess, time, signal, re, threading, logging +import common +import virt_utils + + +class ExpectError(Exception): + def __init__(self, patterns, output): + Exception.__init__(self, patterns, output) + self.patterns = patterns + self.output = output + + def _pattern_str(self): + if len(self.patterns) == 1: + return "pattern %r" % self.patterns[0] + else: + return "patterns %r" % self.patterns + + def __str__(self): + return ("Unknown error occurred while looking for %s (output: %r)" % + (self._pattern_str(), self.output)) + + +class ExpectTimeoutError(ExpectError): + def __str__(self): + return ("Timeout expired while looking for %s (output: %r)" % + (self._pattern_str(), self.output)) + + +class ExpectProcessTerminatedError(ExpectError): + def __init__(self, patterns, status, output): + ExpectError.__init__(self, patterns, output) + self.status = status + + def __str__(self): + return ("Process terminated while looking for %s " + "(status: %s, output: %r)" % (self._pattern_str(), + self.status, self.output)) + + +class ShellError(Exception): + def __init__(self, cmd, output): + Exception.__init__(self, cmd, output) + self.cmd = cmd + self.output = output + + def __str__(self): + return ("Could not execute shell command %r (output: %r)" % + (self.cmd, self.output)) + + +class ShellTimeoutError(ShellError): + def __str__(self): + return ("Timeout expired while waiting for shell command to " + "complete: %r (output: %r)" % (self.cmd, self.output)) + + +class ShellProcessTerminatedError(ShellError): + # Raised when the shell process itself (e.g. ssh, netcat, telnet) + # terminates unexpectedly + def __init__(self, cmd, status, output): + ShellError.__init__(self, cmd, output) + self.status = status + + def __str__(self): + return ("Shell process terminated while waiting for command to " + "complete: %r (status: %s, output: %r)" % + (self.cmd, self.status, self.output)) + + +class ShellCmdError(ShellError): + # Raised when a command executed in a shell terminates with a nonzero + # exit code (status) + def __init__(self, cmd, status, output): + ShellError.__init__(self, cmd, output) + self.status = status + + def __str__(self): + return ("Shell command failed: %r (status: %s, output: %r)" % + (self.cmd, self.status, self.output)) + + +class ShellStatusError(ShellError): + # Raised when the command's exit status cannot be obtained + def __str__(self): + return ("Could not get exit status of command: %r (output: %r)" % + (self.cmd, self.output)) + + +def run_bg(command, termination_func=None, output_func=None, output_prefix="", + timeout=1.0): + """ + Run command as a subprocess. Call output_func with each line of output + from the subprocess (prefixed by output_prefix). Call termination_func + when the subprocess terminates. Return when timeout expires or when the + subprocess exits -- whichever occurs first. + + @brief: Run a subprocess in the background and collect its output and + exit status. + + @param command: The shell command to execute + @param termination_func: A function to call when the process terminates + (should take an integer exit status parameter) + @param output_func: A function to call with each line of output from + the subprocess (should take a string parameter) + @param output_prefix: A string to pre-pend to each line of the output, + before passing it to stdout_func + @param timeout: Time duration (in seconds) to wait for the subprocess to + terminate before returning + + @return: A Tail object. + """ + process = Tail(command=command, + termination_func=termination_func, + output_func=output_func, + output_prefix=output_prefix) + + end_time = time.time() + timeout + while time.time() < end_time and process.is_alive(): + time.sleep(0.1) + + return process + + +def run_fg(command, output_func=None, output_prefix="", timeout=1.0): + """ + Run command as a subprocess. Call output_func with each line of output + from the subprocess (prefixed by prefix). Return when timeout expires or + when the subprocess exits -- whichever occurs first. If timeout expires + and the subprocess is still running, kill it before returning. + + @brief: Run a subprocess in the foreground and collect its output and + exit status. + + @param command: The shell command to execute + @param output_func: A function to call with each line of output from + the subprocess (should take a string parameter) + @param output_prefix: A string to pre-pend to each line of the output, + before passing it to stdout_func + @param timeout: Time duration (in seconds) to wait for the subprocess to + terminate before killing it and returning + + @return: A 2-tuple containing the exit status of the process and its + STDOUT/STDERR output. If timeout expires before the process + terminates, the returned status is None. + """ + process = run_bg(command, None, output_func, output_prefix, timeout) + output = process.get_output() + if process.is_alive(): + status = None + else: + status = process.get_status() + process.close() + return (status, output) + + +class Spawn: + """ + This class is used for spawning and controlling a child process. + + A new instance of this class can either run a new server (a small Python + program that reads output from the child process and reports it to the + client and to a text file) or attach to an already running server. + When a server is started it runs the child process. + The server writes output from the child's STDOUT and STDERR to a text file. + The text file can be accessed at any time using get_output(). + In addition, the server opens as many pipes as requested by the client and + writes the output to them. + The pipes are requested and accessed by classes derived from Spawn. + These pipes are referred to as "readers". + The server also receives input from the client and sends it to the child + process. + An instance of this class can be pickled. Every derived class is + responsible for restoring its own state by properly defining + __getinitargs__(). + + The first named pipe is used by _tail(), a function that runs in the + background and reports new output from the child as it is produced. + The second named pipe is used by a set of functions that read and parse + output as requested by the user in an interactive manner, similar to + pexpect. + When unpickled it automatically + resumes _tail() if needed. + """ + + def __init__(self, command=None, id=None, auto_close=False, echo=False, + linesep="\n"): + """ + Initialize the class and run command as a child process. + + @param command: Command to run, or None if accessing an already running + server. + @param id: ID of an already running server, if accessing a running + server, or None if starting a new one. + @param auto_close: If True, close() the instance automatically when its + reference count drops to zero (default False). + @param echo: Boolean indicating whether echo should be initially + enabled for the pseudo terminal running the subprocess. This + parameter has an effect only when starting a new server. + @param linesep: Line separator to be appended to strings sent to the + child process by sendline(). + """ + self.id = id or virt_utils.generate_random_string(8) + + # Define filenames for communication with server + base_dir = "/tmp/kvm_spawn" + try: + os.makedirs(base_dir) + except: + pass + (self.shell_pid_filename, + self.status_filename, + self.output_filename, + self.inpipe_filename, + self.lock_server_running_filename, + self.lock_client_starting_filename) = _get_filenames(base_dir, + self.id) + + # Remember some attributes + self.auto_close = auto_close + self.echo = echo + self.linesep = linesep + + # Make sure the 'readers' and 'close_hooks' attributes exist + if not hasattr(self, "readers"): + self.readers = [] + if not hasattr(self, "close_hooks"): + self.close_hooks = [] + + # Define the reader filenames + self.reader_filenames = dict( + (reader, _get_reader_filename(base_dir, self.id, reader)) + for reader in self.readers) + + # Let the server know a client intends to open some pipes; + # if the executed command terminates quickly, the server will wait for + # the client to release the lock before exiting + lock_client_starting = _lock(self.lock_client_starting_filename) + + # Start the server (which runs the command) + if command: + sub = subprocess.Popen("%s %s" % (sys.executable, __file__), + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + # Send parameters to the server + sub.stdin.write("%s\n" % self.id) + sub.stdin.write("%s\n" % echo) + sub.stdin.write("%s\n" % ",".join(self.readers)) + sub.stdin.write("%s\n" % command) + # Wait for the server to complete its initialization + while not "Server %s ready" % self.id in sub.stdout.readline(): + pass + + # Open the reading pipes + self.reader_fds = {} + try: + assert(_locked(self.lock_server_running_filename)) + for reader, filename in self.reader_filenames.items(): + self.reader_fds[reader] = os.open(filename, os.O_RDONLY) + except: + pass + + # Allow the server to continue + _unlock(lock_client_starting) + + + # The following two functions are defined to make sure the state is set + # exclusively by the constructor call as specified in __getinitargs__(). + + def __getstate__(self): + pass + + + def __setstate__(self, state): + pass + + + def __getinitargs__(self): + # Save some information when pickling -- will be passed to the + # constructor upon unpickling + return (None, self.id, self.auto_close, self.echo, self.linesep) + + + def __del__(self): + if self.auto_close: + self.close() + + + def _add_reader(self, reader): + """ + Add a reader whose file descriptor can be obtained with _get_fd(). + Should be called before __init__(). Intended for use by derived + classes. + + @param reader: The name of the reader. + """ + if not hasattr(self, "readers"): + self.readers = [] + self.readers.append(reader) + + + def _add_close_hook(self, hook): + """ + Add a close hook function to be called when close() is called. + The function will be called after the process terminates but before + final cleanup. Intended for use by derived classes. + + @param hook: The hook function. + """ + if not hasattr(self, "close_hooks"): + self.close_hooks = [] + self.close_hooks.append(hook) + + + def _get_fd(self, reader): + """ + Return an open file descriptor corresponding to the specified reader + pipe. If no such reader exists, or the pipe could not be opened, + return None. Intended for use by derived classes. + + @param reader: The name of the reader. + """ + return self.reader_fds.get(reader) + + + def get_id(self): + """ + Return the instance's id attribute, which may be used to access the + process in the future. + """ + return self.id + + + def get_pid(self): + """ + Return the PID of the process. + + Note: this may be the PID of the shell process running the user given + command. + """ + try: + file = open(self.shell_pid_filename, "r") + pid = int(file.read()) + file.close() + return pid + except: + return None + + + def get_status(self): + """ + Wait for the process to exit and return its exit status, or None + if the exit status is not available. + """ + _wait(self.lock_server_running_filename) + try: + file = open(self.status_filename, "r") + status = int(file.read()) + file.close() + return status + except: + return None + + + def get_output(self): + """ + Return the STDOUT and STDERR output of the process so far. + """ + try: + file = open(self.output_filename, "r") + output = file.read() + file.close() + return output + except: + return "" + + + def is_alive(self): + """ + Return True if the process is running. + """ + return _locked(self.lock_server_running_filename) + + + def close(self, sig=signal.SIGKILL): + """ + Kill the child process if it's alive and remove temporary files. + + @param sig: The signal to send the process when attempting to kill it. + """ + # Kill it if it's alive + if self.is_alive(): + virt_utils.kill_process_tree(self.get_pid(), sig) + # Wait for the server to exit + _wait(self.lock_server_running_filename) + # Call all cleanup routines + for hook in self.close_hooks: + hook(self) + # Close reader file descriptors + for fd in self.reader_fds.values(): + try: + os.close(fd) + except: + pass + self.reader_fds = {} + # Remove all used files + for filename in (_get_filenames("/tmp/kvm_spawn", self.id) + + self.reader_filenames.values()): + try: + os.unlink(filename) + except OSError: + pass + + + def set_linesep(self, linesep): + """ + Sets the line separator string (usually "\\n"). + + @param linesep: Line separator string. + """ + self.linesep = linesep + + + def send(self, str=""): + """ + Send a string to the child process. + + @param str: String to send to the child process. + """ + try: + fd = os.open(self.inpipe_filename, os.O_RDWR) + os.write(fd, str) + os.close(fd) + except: + pass + + + def sendline(self, str=""): + """ + Send a string followed by a line separator to the child process. + + @param str: String to send to the child process. + """ + self.send(str + self.linesep) + + +_thread_kill_requested = False + +def kill_tail_threads(): + """ + Kill all Tail threads. + + After calling this function no new threads should be started. + """ + global _thread_kill_requested + _thread_kill_requested = True + for t in threading.enumerate(): + if hasattr(t, "name") and t.name.startswith("tail_thread"): + t.join(10) + _thread_kill_requested = False + + +class Tail(Spawn): + """ + This class runs a child process in the background and sends its output in + real time, line-by-line, to a callback function. + + See Spawn's docstring. + + This class uses a single pipe reader to read data in real time from the + child process and report it to a given callback function. + When the child process exits, its exit status is reported to an additional + callback function. + + When this class is unpickled, it automatically resumes reporting output. + """ + + def __init__(self, command=None, id=None, auto_close=False, echo=False, + linesep="\n", termination_func=None, termination_params=(), + output_func=None, output_params=(), output_prefix=""): + """ + Initialize the class and run command as a child process. + + @param command: Command to run, or None if accessing an already running + server. + @param id: ID of an already running server, if accessing a running + server, or None if starting a new one. + @param auto_close: If True, close() the instance automatically when its + reference count drops to zero (default False). + @param echo: Boolean indicating whether echo should be initially + enabled for the pseudo terminal running the subprocess. This + parameter has an effect only when starting a new server. + @param linesep: Line separator to be appended to strings sent to the + child process by sendline(). + @param termination_func: Function to call when the process exits. The + function must accept a single exit status parameter. + @param termination_params: Parameters to send to termination_func + before the exit status. + @param output_func: Function to call whenever a line of output is + available from the STDOUT or STDERR streams of the process. + The function must accept a single string parameter. The string + does not include the final newline. + @param output_params: Parameters to send to output_func before the + output line. + @param output_prefix: String to prepend to lines sent to output_func. + """ + # Add a reader and a close hook + self._add_reader("tail") + self._add_close_hook(Tail._join_thread) + + # Init the superclass + Spawn.__init__(self, command, id, auto_close, echo, linesep) + + # Remember some attributes + self.termination_func = termination_func + self.termination_params = termination_params + self.output_func = output_func + self.output_params = output_params + self.output_prefix = output_prefix + + # Start the thread in the background + self.tail_thread = None + if termination_func or output_func: + self._start_thread() + + + def __getinitargs__(self): + return Spawn.__getinitargs__(self) + (self.termination_func, + self.termination_params, + self.output_func, + self.output_params, + self.output_prefix) + + + def set_termination_func(self, termination_func): + """ + Set the termination_func attribute. See __init__() for details. + + @param termination_func: Function to call when the process terminates. + Must take a single parameter -- the exit status. + """ + self.termination_func = termination_func + if termination_func and not self.tail_thread: + self._start_thread() + + + def set_termination_params(self, termination_params): + """ + Set the termination_params attribute. See __init__() for details. + + @param termination_params: Parameters to send to termination_func + before the exit status. + """ + self.termination_params = termination_params + + + def set_output_func(self, output_func): + """ + Set the output_func attribute. See __init__() for details. + + @param output_func: Function to call for each line of STDOUT/STDERR + output from the process. Must take a single string parameter. + """ + self.output_func = output_func + if output_func and not self.tail_thread: + self._start_thread() + + + def set_output_params(self, output_params): + """ + Set the output_params attribute. See __init__() for details. + + @param output_params: Parameters to send to output_func before the + output line. + """ + self.output_params = output_params + + + def set_output_prefix(self, output_prefix): + """ + Set the output_prefix attribute. See __init__() for details. + + @param output_prefix: String to pre-pend to each line sent to + output_func (see set_output_callback()). + """ + self.output_prefix = output_prefix + + + def _tail(self): + def print_line(text): + # Pre-pend prefix and remove trailing whitespace + text = self.output_prefix + text.rstrip() + # Pass text to output_func + try: + params = self.output_params + (text,) + self.output_func(*params) + except TypeError: + pass + + try: + fd = self._get_fd("tail") + buffer = "" + while True: + global _thread_kill_requested + if _thread_kill_requested: + return + try: + # See if there's any data to read from the pipe + r, w, x = select.select([fd], [], [], 0.05) + except: + break + if fd in r: + # Some data is available; read it + new_data = os.read(fd, 1024) + if not new_data: + break + buffer += new_data + # Send the output to output_func line by line + # (except for the last line) + if self.output_func: + lines = buffer.split("\n") + for line in lines[:-1]: + print_line(line) + # Leave only the last line + last_newline_index = buffer.rfind("\n") + buffer = buffer[last_newline_index+1:] + else: + # No output is available right now; flush the buffer + if buffer: + print_line(buffer) + buffer = "" + # The process terminated; print any remaining output + if buffer: + print_line(buffer) + # Get the exit status, print it and send it to termination_func + status = self.get_status() + if status is None: + return + print_line("(Process terminated with status %s)" % status) + try: + params = self.termination_params + (status,) + self.termination_func(*params) + except TypeError: + pass + finally: + self.tail_thread = None + + + def _start_thread(self): + self.tail_thread = threading.Thread(target=self._tail, + name="tail_thread_%s" % self.id) + self.tail_thread.start() + + + def _join_thread(self): + # Wait for the tail thread to exit + # (it's done this way because self.tail_thread may become None at any + # time) + t = self.tail_thread + if t: + t.join() + + +class Expect(Tail): + """ + This class runs a child process in the background and provides expect-like + services. + + It also provides all of Tail's functionality. + """ + + def __init__(self, command=None, id=None, auto_close=True, echo=False, + linesep="\n", termination_func=None, termination_params=(), + output_func=None, output_params=(), output_prefix=""): + """ + Initialize the class and run command as a child process. + + @param command: Command to run, or None if accessing an already running + server. + @param id: ID of an already running server, if accessing a running + server, or None if starting a new one. + @param auto_close: If True, close() the instance automatically when its + reference count drops to zero (default False). + @param echo: Boolean indicating whether echo should be initially + enabled for the pseudo terminal running the subprocess. This + parameter has an effect only when starting a new server. + @param linesep: Line separator to be appended to strings sent to the + child process by sendline(). + @param termination_func: Function to call when the process exits. The + function must accept a single exit status parameter. + @param termination_params: Parameters to send to termination_func + before the exit status. + @param output_func: Function to call whenever a line of output is + available from the STDOUT or STDERR streams of the process. + The function must accept a single string parameter. The string + does not include the final newline. + @param output_params: Parameters to send to output_func before the + output line. + @param output_prefix: String to prepend to lines sent to output_func. + """ + # Add a reader + self._add_reader("expect") + + # Init the superclass + Tail.__init__(self, command, id, auto_close, echo, linesep, + termination_func, termination_params, + output_func, output_params, output_prefix) + + + def __getinitargs__(self): + return Tail.__getinitargs__(self) + + + def read_nonblocking(self, timeout=None): + """ + Read from child until there is nothing to read for timeout seconds. + + @param timeout: Time (seconds) to wait before we give up reading from + the child process, or None to use the default value. + """ + if timeout is None: + timeout = 0.1 + fd = self._get_fd("expect") + data = "" + while True: + try: + r, w, x = select.select([fd], [], [], timeout) + except: + return data + if fd in r: + new_data = os.read(fd, 1024) + if not new_data: + return data + data += new_data + else: + return data + + + def match_patterns(self, str, patterns): + """ + Match str against a list of patterns. + + Return the index of the first pattern that matches a substring of str. + None and empty strings in patterns are ignored. + If no match is found, return None. + + @param patterns: List of strings (regular expression patterns). + """ + for i in range(len(patterns)): + if not patterns[i]: + continue + if re.search(patterns[i], str): + return i + + + def read_until_output_matches(self, patterns, filter=lambda x: x, + timeout=60, internal_timeout=None, + print_func=None): + """ + Read using read_nonblocking until a match is found using match_patterns, + or until timeout expires. Before attempting to search for a match, the + data is filtered using the filter function provided. + + @brief: Read from child using read_nonblocking until a pattern + matches. + @param patterns: List of strings (regular expression patterns) + @param filter: Function to apply to the data read from the child before + attempting to match it against the patterns (should take and + return a string) + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + @return: Tuple containing the match index and the data read so far + @raise ExpectTimeoutError: Raised if timeout expires + @raise ExpectProcessTerminatedError: Raised if the child process + terminates while waiting for output + @raise ExpectError: Raised if an unknown error occurs + """ + fd = self._get_fd("expect") + o = "" + end_time = time.time() + timeout + while True: + try: + r, w, x = select.select([fd], [], [], + max(0, end_time - time.time())) + except (select.error, TypeError): + break + if not r: + raise ExpectTimeoutError(patterns, o) + # Read data from child + data = self.read_nonblocking(internal_timeout) + if not data: + break + # Print it if necessary + if print_func: + for line in data.splitlines(): + print_func(line) + # Look for patterns + o += data + match = self.match_patterns(filter(o), patterns) + if match is not None: + return match, o + + # Check if the child has terminated + if virt_utils.wait_for(lambda: not self.is_alive(), 5, 0, 0.1): + raise ExpectProcessTerminatedError(patterns, self.get_status(), o) + else: + # This shouldn't happen + raise ExpectError(patterns, o) + + + def read_until_last_word_matches(self, patterns, timeout=60, + internal_timeout=None, print_func=None): + """ + Read using read_nonblocking until the last word of the output matches + one of the patterns (using match_patterns), or until timeout expires. + + @param patterns: A list of strings (regular expression patterns) + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + @return: A tuple containing the match index and the data read so far + @raise ExpectTimeoutError: Raised if timeout expires + @raise ExpectProcessTerminatedError: Raised if the child process + terminates while waiting for output + @raise ExpectError: Raised if an unknown error occurs + """ + def get_last_word(str): + if str: + return str.split()[-1] + else: + return "" + + return self.read_until_output_matches(patterns, get_last_word, + timeout, internal_timeout, + print_func) + + + def read_until_last_line_matches(self, patterns, timeout=60, + internal_timeout=None, print_func=None): + """ + Read using read_nonblocking until the last non-empty line of the output + matches one of the patterns (using match_patterns), or until timeout + expires. Return a tuple containing the match index (or None if no match + was found) and the data read so far. + + @brief: Read using read_nonblocking until the last non-empty line + matches a pattern. + + @param patterns: A list of strings (regular expression patterns) + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + @return: A tuple containing the match index and the data read so far + @raise ExpectTimeoutError: Raised if timeout expires + @raise ExpectProcessTerminatedError: Raised if the child process + terminates while waiting for output + @raise ExpectError: Raised if an unknown error occurs + """ + def get_last_nonempty_line(str): + nonempty_lines = [l for l in str.splitlines() if l.strip()] + if nonempty_lines: + return nonempty_lines[-1] + else: + return "" + + return self.read_until_output_matches(patterns, get_last_nonempty_line, + timeout, internal_timeout, + print_func) + + +class ShellSession(Expect): + """ + This class runs a child process in the background. It it suited for + processes that provide an interactive shell, such as SSH and Telnet. + + It provides all services of Expect and Tail. In addition, it + provides command running services, and a utility function to test the + process for responsiveness. + """ + + def __init__(self, command=None, id=None, auto_close=True, echo=False, + linesep="\n", termination_func=None, termination_params=(), + output_func=None, output_params=(), output_prefix="", + prompt=r"[\#\$]\s*$", status_test_command="echo $?"): + """ + Initialize the class and run command as a child process. + + @param command: Command to run, or None if accessing an already running + server. + @param id: ID of an already running server, if accessing a running + server, or None if starting a new one. + @param auto_close: If True, close() the instance automatically when its + reference count drops to zero (default True). + @param echo: Boolean indicating whether echo should be initially + enabled for the pseudo terminal running the subprocess. This + parameter has an effect only when starting a new server. + @param linesep: Line separator to be appended to strings sent to the + child process by sendline(). + @param termination_func: Function to call when the process exits. The + function must accept a single exit status parameter. + @param termination_params: Parameters to send to termination_func + before the exit status. + @param output_func: Function to call whenever a line of output is + available from the STDOUT or STDERR streams of the process. + The function must accept a single string parameter. The string + does not include the final newline. + @param output_params: Parameters to send to output_func before the + output line. + @param output_prefix: String to prepend to lines sent to output_func. + @param prompt: Regular expression describing the shell's prompt line. + @param status_test_command: Command to be used for getting the last + exit status of commands run inside the shell (used by + cmd_status_output() and friends). + """ + # Init the superclass + Expect.__init__(self, command, id, auto_close, echo, linesep, + termination_func, termination_params, + output_func, output_params, output_prefix) + + # Remember some attributes + self.prompt = prompt + self.status_test_command = status_test_command + + + def __getinitargs__(self): + return Expect.__getinitargs__(self) + (self.prompt, + self.status_test_command) + + + def set_prompt(self, prompt): + """ + Set the prompt attribute for later use by read_up_to_prompt. + + @param: String that describes the prompt contents. + """ + self.prompt = prompt + + + def set_status_test_command(self, status_test_command): + """ + Set the command to be sent in order to get the last exit status. + + @param status_test_command: Command that will be sent to get the last + exit status. + """ + self.status_test_command = status_test_command + + + def is_responsive(self, timeout=5.0): + """ + Return True if the process responds to STDIN/terminal input. + + Send a newline to the child process (e.g. SSH or Telnet) and read some + output using read_nonblocking(). + If all is OK, some output should be available (e.g. the shell prompt). + In that case return True. Otherwise return False. + + @param timeout: Time duration to wait before the process is considered + unresponsive. + """ + # Read all output that's waiting to be read, to make sure the output + # we read next is in response to the newline sent + self.read_nonblocking(timeout=0) + # Send a newline + self.sendline() + # Wait up to timeout seconds for some output from the child + end_time = time.time() + timeout + while time.time() < end_time: + time.sleep(0.5) + if self.read_nonblocking(timeout=0).strip(): + return True + # No output -- report unresponsive + return False + + + def read_up_to_prompt(self, timeout=60, internal_timeout=None, + print_func=None): + """ + Read using read_nonblocking until the last non-empty line of the output + matches the prompt regular expression set by set_prompt, or until + timeout expires. + + @brief: Read using read_nonblocking until the last non-empty line + matches the prompt. + + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being + read (should take a string parameter) + + @return: The data read so far + @raise ExpectTimeoutError: Raised if timeout expires + @raise ExpectProcessTerminatedError: Raised if the shell process + terminates while waiting for output + @raise ExpectError: Raised if an unknown error occurs + """ + m, o = self.read_until_last_line_matches([self.prompt], timeout, + internal_timeout, print_func) + return o + + + def cmd_output(self, cmd, timeout=60, internal_timeout=None, + print_func=None): + """ + Send a command and return its output. + + @param cmd: Command to send (must not contain newline characters) + @param timeout: The duration (in seconds) to wait for the prompt to + return + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + + @return: The output of cmd + @raise ShellTimeoutError: Raised if timeout expires + @raise ShellProcessTerminatedError: Raised if the shell process + terminates while waiting for output + @raise ShellError: Raised if an unknown error occurs + """ + def remove_command_echo(str, cmd): + if str and str.splitlines()[0] == cmd: + str = "".join(str.splitlines(True)[1:]) + return str + + def remove_last_nonempty_line(str): + return "".join(str.rstrip().splitlines(True)[:-1]) + + logging.debug("Sending command: %s" % cmd) + self.read_nonblocking(timeout=0) + self.sendline(cmd) + try: + o = self.read_up_to_prompt(timeout, internal_timeout, print_func) + except ExpectError, e: + o = remove_command_echo(e.output, cmd) + if isinstance(e, ExpectTimeoutError): + raise ShellTimeoutError(cmd, o) + elif isinstance(e, ExpectProcessTerminatedError): + raise ShellProcessTerminatedError(cmd, e.status, o) + else: + raise ShellError(cmd, o) + + # Remove the echoed command and the final shell prompt + return remove_last_nonempty_line(remove_command_echo(o, cmd)) + + + def cmd_status_output(self, cmd, timeout=60, internal_timeout=None, + print_func=None): + """ + Send a command and return its exit status and output. + + @param cmd: Command to send (must not contain newline characters) + @param timeout: The duration (in seconds) to wait for the prompt to + return + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + + @return: A tuple (status, output) where status is the exit status and + output is the output of cmd + @raise ShellTimeoutError: Raised if timeout expires + @raise ShellProcessTerminatedError: Raised if the shell process + terminates while waiting for output + @raise ShellStatusError: Raised if the exit status cannot be obtained + @raise ShellError: Raised if an unknown error occurs + """ + o = self.cmd_output(cmd, timeout, internal_timeout, print_func) + try: + # Send the 'echo $?' (or equivalent) command to get the exit status + s = self.cmd_output(self.status_test_command, 10, internal_timeout) + except ShellError: + raise ShellStatusError(cmd, o) + + # Get the first line consisting of digits only + digit_lines = [l for l in s.splitlines() if l.strip().isdigit()] + if digit_lines: + return int(digit_lines[0].strip()), o + else: + raise ShellStatusError(cmd, o) + + + def cmd_status(self, cmd, timeout=60, internal_timeout=None, + print_func=None): + """ + Send a command and return its exit status. + + @param cmd: Command to send (must not contain newline characters) + @param timeout: The duration (in seconds) to wait for the prompt to + return + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + + @return: The exit status of cmd + @raise ShellTimeoutError: Raised if timeout expires + @raise ShellProcessTerminatedError: Raised if the shell process + terminates while waiting for output + @raise ShellStatusError: Raised if the exit status cannot be obtained + @raise ShellError: Raised if an unknown error occurs + """ + s, o = self.cmd_status_output(cmd, timeout, internal_timeout, + print_func) + return s + + + def cmd(self, cmd, timeout=60, internal_timeout=None, print_func=None): + """ + Send a command and return its output. If the command's exit status is + nonzero, raise an exception. + + @param cmd: Command to send (must not contain newline characters) + @param timeout: The duration (in seconds) to wait for the prompt to + return + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + + @return: The output of cmd + @raise ShellTimeoutError: Raised if timeout expires + @raise ShellProcessTerminatedError: Raised if the shell process + terminates while waiting for output + @raise ShellError: Raised if the exit status cannot be obtained or if + an unknown error occurs + @raise ShellStatusError: Raised if the exit status cannot be obtained + @raise ShellError: Raised if an unknown error occurs + @raise ShellCmdError: Raised if the exit status is nonzero + """ + s, o = self.cmd_status_output(cmd, timeout, internal_timeout, + print_func) + if s != 0: + raise ShellCmdError(cmd, s, o) + return o + + + def get_command_output(self, cmd, timeout=60, internal_timeout=None, + print_func=None): + """ + Alias for cmd_output() for backward compatibility. + """ + return self.cmd_output(cmd, timeout, internal_timeout, print_func) + + + def get_command_status_output(self, cmd, timeout=60, internal_timeout=None, + print_func=None): + """ + Alias for cmd_status_output() for backward compatibility. + """ + return self.cmd_status_output(cmd, timeout, internal_timeout, + print_func) + + + def get_command_status(self, cmd, timeout=60, internal_timeout=None, + print_func=None): + """ + Alias for cmd_status() for backward compatibility. + """ + return self.cmd_status(cmd, timeout, internal_timeout, print_func) diff --git a/client/virt/kvm_installer.py b/client/virt/kvm_installer.py new file mode 100644 index 0000000..ea48e95 --- /dev/null +++ b/client/virt/kvm_installer.py @@ -0,0 +1,797 @@ +import os, logging, datetime, glob +import shutil +from autotest_lib.client.bin import utils, os_dep +from autotest_lib.client.common_lib import error +import virt_utils + + +def check_configure_options(script_path): + """ + Return the list of available options (flags) of a given kvm configure build + script. + + @param script: Path to the configure script + """ + abspath = os.path.abspath(script_path) + help_raw = utils.system_output('%s --help' % abspath, ignore_status=True) + help_output = help_raw.split("\n") + option_list = [] + for line in help_output: + cleaned_line = line.lstrip() + if cleaned_line.startswith("--"): + option = cleaned_line.split()[0] + option = option.split("=")[0] + option_list.append(option) + + return option_list + + +def kill_qemu_processes(): + """ + Kills all qemu processes, also kills all processes holding /dev/kvm down. + """ + logging.debug("Killing any qemu processes that might be left behind") + utils.system("pkill qemu", ignore_status=True) + # Let's double check to see if some other process is holding /dev/kvm + if os.path.isfile("/dev/kvm"): + utils.system("fuser -k /dev/kvm", ignore_status=True) + + +def cpu_vendor(): + vendor = "intel" + if os.system("grep vmx /proc/cpuinfo 1>/dev/null") != 0: + vendor = "amd" + logging.debug("Detected CPU vendor as '%s'", vendor) + return vendor + + +def _unload_kvm_modules(mod_list): + logging.info("Unloading previously loaded KVM modules") + for module in reversed(mod_list): + utils.unload_module(module) + + +def _load_kvm_modules(mod_list, module_dir=None, load_stock=False): + """ + Just load the KVM modules, without killing Qemu or unloading previous + modules. + + Load modules present on any sub directory of module_dir. Function will walk + through module_dir until it finds the modules. + + @param module_dir: Directory where the KVM modules are located. + @param load_stock: Whether we are going to load system kernel modules. + @param extra_modules: List of extra modules to load. + """ + if module_dir: + logging.info("Loading the built KVM modules...") + kvm_module_path = None + kvm_vendor_module_path = None + abort = False + + list_modules = ['%s.ko' % (m) for m in mod_list] + + list_module_paths = [] + for folder, subdirs, files in os.walk(module_dir): + for module in list_modules: + if module in files: + module_path = os.path.join(folder, module) + list_module_paths.append(module_path) + + # We might need to arrange the modules in the correct order + # to avoid module load problems + list_modules_load = [] + for module in list_modules: + for module_path in list_module_paths: + if os.path.basename(module_path) == module: + list_modules_load.append(module_path) + + if len(list_module_paths) != len(list_modules): + logging.error("KVM modules not found. If you don't want to use the " + "modules built by this test, make sure the option " + "load_modules: 'no' is marked on the test control " + "file.") + raise error.TestError("The modules %s were requested to be loaded, " + "but the only modules found were %s" % + (list_modules, list_module_paths)) + + for module_path in list_modules_load: + try: + utils.system("insmod %s" % module_path) + except Exception, e: + raise error.TestFail("Failed to load KVM modules: %s" % e) + + if load_stock: + logging.info("Loading current system KVM modules...") + for module in mod_list: + utils.system("modprobe %s" % module) + + +def create_symlinks(test_bindir, prefix=None, bin_list=None, unittest=None): + """ + Create symbolic links for the appropriate qemu and qemu-img commands on + the kvm test bindir. + + @param test_bindir: KVM test bindir + @param prefix: KVM prefix path + @param bin_list: List of qemu binaries to link + @param unittest: Path to configuration file unittests.cfg + """ + qemu_path = os.path.join(test_bindir, "qemu") + qemu_img_path = os.path.join(test_bindir, "qemu-img") + qemu_unittest_path = os.path.join(test_bindir, "unittests") + if os.path.lexists(qemu_path): + os.unlink(qemu_path) + if os.path.lexists(qemu_img_path): + os.unlink(qemu_img_path) + if unittest and os.path.lexists(qemu_unittest_path): + os.unlink(qemu_unittest_path) + + logging.debug("Linking qemu binaries") + + if bin_list: + for bin in bin_list: + if os.path.basename(bin) == 'qemu-kvm': + os.symlink(bin, qemu_path) + elif os.path.basename(bin) == 'qemu-img': + os.symlink(bin, qemu_img_path) + + elif prefix: + kvm_qemu = os.path.join(prefix, "bin", "qemu-system-x86_64") + if not os.path.isfile(kvm_qemu): + raise error.TestError('Invalid qemu path') + kvm_qemu_img = os.path.join(prefix, "bin", "qemu-img") + if not os.path.isfile(kvm_qemu_img): + raise error.TestError('Invalid qemu-img path') + os.symlink(kvm_qemu, qemu_path) + os.symlink(kvm_qemu_img, qemu_img_path) + + if unittest: + logging.debug("Linking unittest dir") + os.symlink(unittest, qemu_unittest_path) + + +def install_roms(rom_dir, prefix): + logging.debug("Path to roms specified. Copying roms to install prefix") + rom_dst_dir = os.path.join(prefix, 'share', 'qemu') + for rom_src in glob.glob('%s/*.bin' % rom_dir): + rom_dst = os.path.join(rom_dst_dir, os.path.basename(rom_src)) + logging.debug("Copying rom file %s to %s", rom_src, rom_dst) + shutil.copy(rom_src, rom_dst) + + +def save_build(build_dir, dest_dir): + logging.debug('Saving the result of the build on %s', dest_dir) + base_name = os.path.basename(build_dir) + tarball_name = base_name + '.tar.bz2' + os.chdir(os.path.dirname(build_dir)) + utils.system('tar -cjf %s %s' % (tarball_name, base_name)) + shutil.move(tarball_name, os.path.join(dest_dir, tarball_name)) + + +class KvmInstallException(Exception): + pass + + +class FailedKvmInstall(KvmInstallException): + pass + + +class KvmNotInstalled(KvmInstallException): + pass + + +class BaseInstaller(object): + # default value for load_stock argument + load_stock_modules = True + def __init__(self, mode=None): + self.install_mode = mode + self._full_module_list = None + + def set_install_params(self, test, params): + self.params = params + + load_modules = params.get('load_modules', 'no') + if not load_modules or load_modules == 'yes': + self.should_load_modules = True + elif load_modules == 'no': + self.should_load_modules = False + default_extra_modules = str(None) + self.extra_modules = eval(params.get("extra_modules", + default_extra_modules)) + + self.cpu_vendor = cpu_vendor() + + self.srcdir = test.srcdir + if not os.path.isdir(self.srcdir): + os.makedirs(self.srcdir) + + self.test_bindir = test.bindir + self.results_dir = test.resultsdir + + # KVM build prefix, for the modes that do need it + prefix = os.path.join(test.bindir, 'build') + self.prefix = os.path.abspath(prefix) + + # Current host kernel directory + default_host_kernel_source = '/lib/modules/%s/build' % os.uname()[2] + self.host_kernel_srcdir = params.get('host_kernel_source', + default_host_kernel_source) + + # Extra parameters that can be passed to the configure script + self.extra_configure_options = params.get('extra_configure_options', + None) + + # Do we want to save the result of the build on test.resultsdir? + self.save_results = True + save_results = params.get('save_results', 'no') + if save_results == 'no': + self.save_results = False + + self._full_module_list = list(self._module_list()) + + + def install_unittests(self): + userspace_srcdir = os.path.join(self.srcdir, "kvm_userspace") + test_repo = self.params.get("test_git_repo") + test_branch = self.params.get("test_branch", "master") + test_commit = self.params.get("test_commit", None) + test_lbranch = self.params.get("test_lbranch", "master") + + if test_repo: + test_srcdir = os.path.join(self.srcdir, "kvm-unit-tests") + virt_utils.get_git_branch(test_repo, test_branch, test_srcdir, + test_commit, test_lbranch) + unittest_cfg = os.path.join(test_srcdir, 'x86', + 'unittests.cfg') + self.test_srcdir = test_srcdir + else: + unittest_cfg = os.path.join(userspace_srcdir, 'kvm', 'test', 'x86', + 'unittests.cfg') + self.unittest_cfg = None + if os.path.isfile(unittest_cfg): + self.unittest_cfg = unittest_cfg + else: + if test_repo: + logging.error("No unittest config file %s found, skipping " + "unittest build", self.unittest_cfg) + + self.unittest_prefix = None + if self.unittest_cfg: + logging.info("Building and installing unittests") + os.chdir(os.path.dirname(os.path.dirname(self.unittest_cfg))) + utils.system('./configure --prefix=%s' % self.prefix) + utils.system('make') + utils.system('make install') + self.unittest_prefix = os.path.join(self.prefix, 'share', 'qemu', + 'tests') + + + def full_module_list(self): + """Return the module list used by the installer + + Used by the module_probe test, to avoid using utils.unload_module(). + """ + if self._full_module_list is None: + raise KvmNotInstalled("KVM modules not installed yet (installer: %s)" % (type(self))) + return self._full_module_list + + + def _module_list(self): + """Generate the list of modules that need to be loaded + """ + yield 'kvm' + yield 'kvm-%s' % (self.cpu_vendor) + if self.extra_modules: + for module in self.extra_modules: + yield module + + + def _load_modules(self, mod_list): + """ + Load the KVM modules + + May be overridden by subclasses. + """ + _load_kvm_modules(mod_list, load_stock=self.load_stock_modules) + + + def load_modules(self, mod_list=None): + if mod_list is None: + mod_list = self.full_module_list() + self._load_modules(mod_list) + + + def _unload_modules(self, mod_list=None): + """ + Just unload the KVM modules, without trying to kill Qemu + """ + if mod_list is None: + mod_list = self.full_module_list() + _unload_kvm_modules(mod_list) + + + def unload_modules(self, mod_list=None): + """ + Kill Qemu and unload the KVM modules + """ + kill_qemu_processes() + self._unload_modules(mod_list) + + + def reload_modules(self): + """ + Reload the KVM modules after killing Qemu and unloading the current modules + """ + self.unload_modules() + self.load_modules() + + + def reload_modules_if_needed(self): + if self.should_load_modules: + self.reload_modules() + + +class YumInstaller(BaseInstaller): + """ + Class that uses yum to install and remove packages. + """ + load_stock_modules = True + def set_install_params(self, test, params): + super(YumInstaller, self).set_install_params(test, params) + # Checking if all required dependencies are available + os_dep.command("rpm") + os_dep.command("yum") + + default_pkg_list = str(['qemu-kvm', 'qemu-kvm-tools']) + default_qemu_bin_paths = str(['/usr/bin/qemu-kvm', '/usr/bin/qemu-img']) + default_pkg_path_list = str(None) + self.pkg_list = eval(params.get("pkg_list", default_pkg_list)) + self.pkg_path_list = eval(params.get("pkg_path_list", + default_pkg_path_list)) + self.qemu_bin_paths = eval(params.get("qemu_bin_paths", + default_qemu_bin_paths)) + + + def _clean_previous_installs(self): + kill_qemu_processes() + removable_packages = "" + for pkg in self.pkg_list: + removable_packages += " %s" % pkg + + utils.system("yum remove -y %s" % removable_packages) + + + def _get_packages(self): + for pkg in self.pkg_path_list: + utils.get_file(pkg, os.path.join(self.srcdir, + os.path.basename(pkg))) + + + def _install_packages(self): + """ + Install all downloaded packages. + """ + os.chdir(self.srcdir) + utils.system("yum install --nogpgcheck -y *.rpm") + + + def install(self): + self.install_unittests() + self._clean_previous_installs() + self._get_packages() + self._install_packages() + create_symlinks(test_bindir=self.test_bindir, + bin_list=self.qemu_bin_paths, + unittest=self.unittest_prefix) + self.reload_modules_if_needed() + if self.save_results: + save_build(self.srcdir, self.results_dir) + + +class KojiInstaller(YumInstaller): + """ + Class that handles installing KVM from the fedora build service, koji. + It uses yum to install and remove packages. + """ + load_stock_modules = True + def set_install_params(self, test, params): + """ + Gets parameters and initializes the package downloader. + + @param test: kvm test object + @param params: Dictionary with test arguments + """ + super(KojiInstaller, self).set_install_params(test, params) + default_koji_cmd = '/usr/bin/koji' + default_src_pkg = 'qemu' + self.src_pkg = params.get("src_pkg", default_src_pkg) + self.tag = params.get("koji_tag", None) + self.build = params.get("koji_build", None) + self.koji_cmd = params.get("koji_cmd", default_koji_cmd) + + + def _get_packages(self): + """ + Downloads the specific arch RPMs for the specific build name. + """ + downloader = virt_utils.KojiDownloader(cmd=self.koji_cmd) + downloader.get(src_package=self.src_pkg, tag=self.tag, + build=self.build, dst_dir=self.srcdir) + + + def install(self): + super(KojiInstaller, self)._clean_previous_installs() + self._get_packages() + super(KojiInstaller, self)._install_packages() + self.install_unittests() + create_symlinks(test_bindir=self.test_bindir, + bin_list=self.qemu_bin_paths, + unittest=self.unittest_prefix) + self.reload_modules_if_needed() + if self.save_results: + save_build(self.srcdir, self.results_dir) + + +class SourceDirInstaller(BaseInstaller): + """ + Class that handles building/installing KVM directly from a tarball or + a single source code dir. + """ + def set_install_params(self, test, params): + """ + Initializes class attributes, and retrieves KVM code. + + @param test: kvm test object + @param params: Dictionary with test arguments + """ + super(SourceDirInstaller, self).set_install_params(test, params) + + self.mod_install_dir = os.path.join(self.prefix, 'modules') + self.installed_kmods = False # it will be set to True in case we + # installed our own modules + + srcdir = params.get("srcdir", None) + self.path_to_roms = params.get("path_to_rom_images", None) + + if self.install_mode == 'localsrc': + if srcdir is None: + raise error.TestError("Install from source directory specified" + "but no source directory provided on the" + "control file.") + else: + shutil.copytree(srcdir, self.srcdir) + + if self.install_mode == 'release': + release_tag = params.get("release_tag") + release_dir = params.get("release_dir") + release_listing = params.get("release_listing") + logging.info("Installing KVM from release tarball") + if not release_tag: + release_tag = virt_utils.get_latest_kvm_release_tag( + release_listing) + tarball = os.path.join(release_dir, 'kvm', release_tag, + "kvm-%s.tar.gz" % release_tag) + logging.info("Retrieving release kvm-%s" % release_tag) + tarball = utils.unmap_url("/", tarball, "/tmp") + + elif self.install_mode == 'snapshot': + logging.info("Installing KVM from snapshot") + snapshot_dir = params.get("snapshot_dir") + if not snapshot_dir: + raise error.TestError("Snapshot dir not provided") + snapshot_date = params.get("snapshot_date") + if not snapshot_date: + # Take yesterday's snapshot + d = (datetime.date.today() - + datetime.timedelta(1)).strftime("%Y%m%d") + else: + d = snapshot_date + tarball = os.path.join(snapshot_dir, "kvm-snapshot-%s.tar.gz" % d) + logging.info("Retrieving kvm-snapshot-%s" % d) + tarball = utils.unmap_url("/", tarball, "/tmp") + + elif self.install_mode == 'localtar': + tarball = params.get("tarball") + if not tarball: + raise error.TestError("KVM Tarball install specified but no" + " tarball provided on control file.") + logging.info("Installing KVM from a local tarball") + logging.info("Using tarball %s") + tarball = utils.unmap_url("/", params.get("tarball"), "/tmp") + + if self.install_mode in ['release', 'snapshot', 'localtar']: + utils.extract_tarball_to_dir(tarball, self.srcdir) + + if self.install_mode in ['release', 'snapshot', 'localtar', 'srcdir']: + self.repo_type = virt_utils.check_kvm_source_dir(self.srcdir) + configure_script = os.path.join(self.srcdir, 'configure') + self.configure_options = check_configure_options(configure_script) + + + def _build(self): + make_jobs = utils.count_cpus() + os.chdir(self.srcdir) + # For testing purposes, it's better to build qemu binaries with + # debugging symbols, so we can extract more meaningful stack traces. + cfg = "./configure --prefix=%s" % self.prefix + if "--disable-strip" in self.configure_options: + cfg += " --disable-strip" + steps = [cfg, "make clean", "make -j %s" % make_jobs] + logging.info("Building KVM") + for step in steps: + utils.system(step) + + + def _install_kmods_old_userspace(self, userspace_path): + """ + Run the module install command. + + This is for the "old userspace" code, that contained a 'kernel' subdirectory + with the kmod build code. + + The code would be much simpler if we could specify the module install + path as parameter to the toplevel Makefile. As we can't do that and + the module install code doesn't use --prefix, we have to call + 'make -C kernel install' directly, setting the module directory + parameters. + + If the userspace tree doens't have a 'kernel' subdirectory, the + module install step will be skipped. + + @param userspace_path: the path the kvm-userspace directory + """ + kdir = os.path.join(userspace_path, 'kernel') + if os.path.isdir(kdir): + os.chdir(kdir) + # INSTALLDIR is the target dir for the modules + # ORIGMODDIR is the dir where the old modules will be removed. we + # don't want to mess with the system modules, so set it + # to a non-existing directory + utils.system('make install INSTALLDIR=%s ORIGMODDIR=/tmp/no-old-modules' % (self.mod_install_dir)) + self.installed_kmods = True + + + def _install_kmods(self, kmod_path): + """Run the module install command for the kmod-kvm repository + + @param kmod_path: the path to the kmod-kvm.git working copy + """ + os.chdir(kmod_path) + utils.system('make modules_install DESTDIR=%s' % (self.mod_install_dir)) + self.installed_kmods = True + + + def _install(self): + os.chdir(self.srcdir) + logging.info("Installing KVM userspace") + if self.repo_type == 1: + utils.system("make -C qemu install") + self._install_kmods_old_userspace(self.srcdir) + elif self.repo_type == 2: + utils.system("make install") + if self.path_to_roms: + install_roms(self.path_to_roms, self.prefix) + self.install_unittests() + create_symlinks(test_bindir=self.test_bindir, + prefix=self.prefix, + unittest=self.unittest_prefix) + + + def _load_modules(self, mod_list): + # load the installed KVM modules in case we installed them + # ourselves. Otherwise, just load the system modules. + if self.installed_kmods: + logging.info("Loading installed KVM modules") + _load_kvm_modules(mod_list, module_dir=self.mod_install_dir) + else: + logging.info("Loading stock KVM modules") + _load_kvm_modules(mod_list, load_stock=True) + + + def install(self): + self._build() + self._install() + self.reload_modules_if_needed() + if self.save_results: + save_build(self.srcdir, self.results_dir) + + +class GitInstaller(SourceDirInstaller): + def _pull_code(self): + """ + Retrieves code from git repositories. + """ + params = self.params + + kernel_repo = params.get("git_repo") + user_repo = params.get("user_git_repo") + kmod_repo = params.get("kmod_repo") + + kernel_branch = params.get("kernel_branch", "master") + user_branch = params.get("user_branch", "master") + kmod_branch = params.get("kmod_branch", "master") + + kernel_lbranch = params.get("kernel_lbranch", "master") + user_lbranch = params.get("user_lbranch", "master") + kmod_lbranch = params.get("kmod_lbranch", "master") + + kernel_commit = params.get("kernel_commit", None) + user_commit = params.get("user_commit", None) + kmod_commit = params.get("kmod_commit", None) + + kernel_patches = eval(params.get("kernel_patches", "[]")) + user_patches = eval(params.get("user_patches", "[]")) + kmod_patches = eval(params.get("user_patches", "[]")) + + if not user_repo: + message = "KVM user git repository path not specified" + logging.error(message) + raise error.TestError(message) + + userspace_srcdir = os.path.join(self.srcdir, "kvm_userspace") + virt_utils.get_git_branch(user_repo, user_branch, userspace_srcdir, + user_commit, user_lbranch) + self.userspace_srcdir = userspace_srcdir + + if user_patches: + os.chdir(self.userspace_srcdir) + for patch in user_patches: + utils.get_file(patch, os.path.join(self.userspace_srcdir, + os.path.basename(patch))) + utils.system('patch -p1 %s' % os.path.basename(patch)) + + if kernel_repo: + kernel_srcdir = os.path.join(self.srcdir, "kvm") + virt_utils.get_git_branch(kernel_repo, kernel_branch, kernel_srcdir, + kernel_commit, kernel_lbranch) + self.kernel_srcdir = kernel_srcdir + if kernel_patches: + os.chdir(self.kernel_srcdir) + for patch in kernel_patches: + utils.get_file(patch, os.path.join(self.userspace_srcdir, + os.path.basename(patch))) + utils.system('patch -p1 %s' % os.path.basename(patch)) + else: + self.kernel_srcdir = None + + if kmod_repo: + kmod_srcdir = os.path.join (self.srcdir, "kvm_kmod") + virt_utils.get_git_branch(kmod_repo, kmod_branch, kmod_srcdir, + kmod_commit, kmod_lbranch) + self.kmod_srcdir = kmod_srcdir + if kmod_patches: + os.chdir(self.kmod_srcdir) + for patch in kmod_patches: + utils.get_file(patch, os.path.join(self.userspace_srcdir, + os.path.basename(patch))) + utils.system('patch -p1 %s' % os.path.basename(patch)) + else: + self.kmod_srcdir = None + + configure_script = os.path.join(self.userspace_srcdir, 'configure') + self.configure_options = check_configure_options(configure_script) + + + def _build(self): + make_jobs = utils.count_cpus() + cfg = './configure' + if self.kmod_srcdir: + logging.info('Building KVM modules') + os.chdir(self.kmod_srcdir) + module_build_steps = [cfg, + 'make clean', + 'make sync LINUX=%s' % self.kernel_srcdir, + 'make'] + elif self.kernel_srcdir: + logging.info('Building KVM modules') + os.chdir(self.userspace_srcdir) + cfg += ' --kerneldir=%s' % self.host_kernel_srcdir + module_build_steps = [cfg, + 'make clean', + 'make -C kernel LINUX=%s sync' % self.kernel_srcdir] + else: + module_build_steps = [] + + for step in module_build_steps: + utils.run(step) + + logging.info('Building KVM userspace code') + os.chdir(self.userspace_srcdir) + cfg += ' --prefix=%s' % self.prefix + if "--disable-strip" in self.configure_options: + cfg += ' --disable-strip' + if self.extra_configure_options: + cfg += ' %s' % self.extra_configure_options + utils.system(cfg) + utils.system('make clean') + utils.system('make -j %s' % make_jobs) + + + def _install(self): + if self.kernel_srcdir: + os.chdir(self.userspace_srcdir) + # the kernel module install with --prefix doesn't work, and DESTDIR + # wouldn't work for the userspace stuff, so we clear WANT_MODULE: + utils.system('make install WANT_MODULE=') + # and install the old-style-kmod modules manually: + self._install_kmods_old_userspace(self.userspace_srcdir) + elif self.kmod_srcdir: + # if we have a kmod repository, it is easier: + # 1) install userspace: + os.chdir(self.userspace_srcdir) + utils.system('make install') + # 2) install kmod: + self._install_kmods(self.kmod_srcdir) + else: + # if we don't have kmod sources, we just install + # userspace: + os.chdir(self.userspace_srcdir) + utils.system('make install') + + if self.path_to_roms: + install_roms(self.path_to_roms, self.prefix) + self.install_unittests() + create_symlinks(test_bindir=self.test_bindir, prefix=self.prefix, + bin_list=None, + unittest=self.unittest_prefix) + + + def install(self): + self._pull_code() + self._build() + self._install() + self.reload_modules_if_needed() + if self.save_results: + save_build(self.srcdir, self.results_dir) + + +class PreInstalledKvm(BaseInstaller): + # load_modules() will use the stock modules: + load_stock_modules = True + def install(self): + logging.info("Expecting KVM to be already installed. Doing nothing") + + +class FailedInstaller: + """ + Class used to be returned instead of the installer if a installation fails + + Useful to make sure no installer object is used if KVM installation fails. + """ + def __init__(self, msg="KVM install failed"): + self._msg = msg + + + def load_modules(self): + """Will refuse to load the KVM modules as install failed""" + raise FailedKvmInstall("KVM modules not available. reason: %s" % (self._msg)) + + +installer_classes = { + 'localsrc': SourceDirInstaller, + 'localtar': SourceDirInstaller, + 'release': SourceDirInstaller, + 'snapshot': SourceDirInstaller, + 'git': GitInstaller, + 'yum': YumInstaller, + 'koji': KojiInstaller, + 'preinstalled': PreInstalledKvm, +} + + +def _installer_class(install_mode): + c = installer_classes.get(install_mode) + if c is None: + raise error.TestError('Invalid or unsupported' + ' install mode: %s' % install_mode) + return c + + +def make_installer(params): + # priority: + # - 'install_mode' param + # - 'mode' param + mode = params.get("install_mode", params.get("mode")) + klass = _installer_class(mode) + return klass(mode) diff --git a/client/virt/kvm_monitor.py b/client/virt/kvm_monitor.py new file mode 100644 index 0000000..d76f5c2 --- /dev/null +++ b/client/virt/kvm_monitor.py @@ -0,0 +1,745 @@ +""" +Interfaces to the QEMU monitor. + +@copyright: 2008-2010 Red Hat Inc. +""" + +import socket, time, threading, logging, select +import virt_utils + +try: + import json +except ImportError: + logging.warning("Could not import json module. " + "QMP monitor functionality disabled.") + + +class MonitorError(Exception): + pass + + +class MonitorConnectError(MonitorError): + pass + + +class MonitorSocketError(MonitorError): + def __init__(self, msg, e): + Exception.__init__(self, msg, e) + self.msg = msg + self.e = e + + def __str__(self): + return "%s (%s)" % (self.msg, self.e) + + +class MonitorLockError(MonitorError): + pass + + +class MonitorProtocolError(MonitorError): + pass + + +class MonitorNotSupportedError(MonitorError): + pass + + +class QMPCmdError(MonitorError): + def __init__(self, cmd, qmp_args, data): + MonitorError.__init__(self, cmd, qmp_args, data) + self.cmd = cmd + self.qmp_args = qmp_args + self.data = data + + def __str__(self): + return ("QMP command %r failed (arguments: %r, " + "error message: %r)" % (self.cmd, self.qmp_args, self.data)) + + +class Monitor: + """ + Common code for monitor classes. + """ + + def __init__(self, name, filename): + """ + Initialize the instance. + + @param name: Monitor identifier (a string) + @param filename: Monitor socket filename + @raise MonitorConnectError: Raised if the connection fails + """ + self.name = name + self.filename = filename + self._lock = threading.RLock() + self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + try: + self._socket.connect(filename) + except socket.error: + raise MonitorConnectError("Could not connect to monitor socket") + + + def __del__(self): + # Automatically close the connection when the instance is garbage + # collected + try: + self._socket.shutdown(socket.SHUT_RDWR) + except socket.error: + pass + self._socket.close() + + + # The following two functions are defined to make sure the state is set + # exclusively by the constructor call as specified in __getinitargs__(). + + def __getstate__(self): + pass + + + def __setstate__(self, state): + pass + + + def __getinitargs__(self): + # Save some information when pickling -- will be passed to the + # constructor upon unpickling + return self.name, self.filename, True + + + def _acquire_lock(self, timeout=20): + end_time = time.time() + timeout + while time.time() < end_time: + if self._lock.acquire(False): + return True + time.sleep(0.05) + return False + + + def _data_available(self, timeout=0): + timeout = max(0, timeout) + return bool(select.select([self._socket], [], [], timeout)[0]) + + + def _recvall(self): + s = "" + while self._data_available(): + try: + data = self._socket.recv(1024) + except socket.error, e: + raise MonitorSocketError("Could not receive data from monitor", + e) + if not data: + break + s += data + return s + + + def is_responsive(self): + """ + Return True iff the monitor is responsive. + """ + try: + self.verify_responsive() + return True + except MonitorError: + return False + + +class HumanMonitor(Monitor): + """ + Wraps "human monitor" commands. + """ + + def __init__(self, name, filename, suppress_exceptions=False): + """ + Connect to the monitor socket and find the (qemu) prompt. + + @param name: Monitor identifier (a string) + @param filename: Monitor socket filename + @raise MonitorConnectError: Raised if the connection fails and + suppress_exceptions is False + @raise MonitorProtocolError: Raised if the initial (qemu) prompt isn't + found and suppress_exceptions is False + @note: Other exceptions may be raised. See cmd()'s + docstring. + """ + try: + Monitor.__init__(self, name, filename) + + self.protocol = "human" + + # Find the initial (qemu) prompt + s, o = self._read_up_to_qemu_prompt(20) + if not s: + raise MonitorProtocolError("Could not find (qemu) prompt " + "after connecting to monitor. " + "Output so far: %r" % o) + + # Save the output of 'help' for future use + self._help_str = self.cmd("help") + + except MonitorError, e: + if suppress_exceptions: + logging.warn(e) + else: + raise + + + # Private methods + + def _read_up_to_qemu_prompt(self, timeout=20): + s = "" + end_time = time.time() + timeout + while self._data_available(end_time - time.time()): + data = self._recvall() + if not data: + break + s += data + try: + if s.splitlines()[-1].split()[-1] == "(qemu)": + return True, "\n".join(s.splitlines()[:-1]) + except IndexError: + continue + return False, "\n".join(s.splitlines()) + + + def _send(self, cmd): + """ + Send a command without waiting for output. + + @param cmd: Command to send + @raise MonitorLockError: Raised if the lock cannot be acquired + @raise MonitorSocketError: Raised if a socket error occurs + """ + if not self._acquire_lock(20): + raise MonitorLockError("Could not acquire exclusive lock to send " + "monitor command '%s'" % cmd) + + try: + try: + self._socket.sendall(cmd + "\n") + except socket.error, e: + raise MonitorSocketError("Could not send monitor command %r" % + cmd, e) + + finally: + self._lock.release() + + + # Public methods + + def cmd(self, command, timeout=20): + """ + Send command to the monitor. + + @param command: Command to send to the monitor + @param timeout: Time duration to wait for the (qemu) prompt to return + @return: Output received from the monitor + @raise MonitorLockError: Raised if the lock cannot be acquired + @raise MonitorSocketError: Raised if a socket error occurs + @raise MonitorProtocolError: Raised if the (qemu) prompt cannot be + found after sending the command + """ + if not self._acquire_lock(20): + raise MonitorLockError("Could not acquire exclusive lock to send " + "monitor command '%s'" % command) + + try: + # Read any data that might be available + self._recvall() + # Send command + self._send(command) + # Read output + s, o = self._read_up_to_qemu_prompt(timeout) + # Remove command echo from output + o = "\n".join(o.splitlines()[1:]) + # Report success/failure + if s: + return o + else: + msg = ("Could not find (qemu) prompt after command '%s'. " + "Output so far: %r" % (command, o)) + raise MonitorProtocolError(msg) + + finally: + self._lock.release() + + + def verify_responsive(self): + """ + Make sure the monitor is responsive by sending a command. + """ + self.cmd("info status") + + + # Command wrappers + # Notes: + # - All of the following commands raise exceptions in a similar manner to + # cmd(). + # - A command wrapper should use self._help_str if it requires information + # about the monitor's capabilities. + + def quit(self): + """ + Send "quit" without waiting for output. + """ + self._send("quit") + + + def info(self, what): + """ + Request info about something and return the output. + """ + return self.cmd("info %s" % what) + + + def query(self, what): + """ + Alias for info. + """ + return self.info(what) + + + def screendump(self, filename): + """ + Request a screendump. + + @param filename: Location for the screendump + @return: The command's output + """ + return self.cmd("screendump %s" % filename) + + + def migrate(self, uri, full_copy=False, incremental_copy=False, wait=False): + """ + Migrate. + + @param uri: destination URI + @param full_copy: If true, migrate with full disk copy + @param incremental_copy: If true, migrate with incremental disk copy + @param wait: If true, wait for completion + @return: The command's output + """ + cmd = "migrate" + if not wait: + cmd += " -d" + if full_copy: + cmd += " -b" + if incremental_copy: + cmd += " -i" + cmd += " %s" % uri + return self.cmd(cmd) + + + def migrate_set_speed(self, value): + """ + Set maximum speed (in bytes/sec) for migrations. + + @param value: Speed in bytes/sec + @return: The command's output + """ + return self.cmd("migrate_set_speed %s" % value) + + + def sendkey(self, keystr, hold_time=1): + """ + Send key combination to VM. + + @param keystr: Key combination string + @param hold_time: Hold time in ms (should normally stay 1 ms) + @return: The command's output + """ + return self.cmd("sendkey %s %s" % (keystr, hold_time)) + + + def mouse_move(self, dx, dy): + """ + Move mouse. + + @param dx: X amount + @param dy: Y amount + @return: The command's output + """ + return self.cmd("mouse_move %d %d" % (dx, dy)) + + + def mouse_button(self, state): + """ + Set mouse button state. + + @param state: Button state (1=L, 2=M, 4=R) + @return: The command's output + """ + return self.cmd("mouse_button %d" % state) + + +class QMPMonitor(Monitor): + """ + Wraps QMP monitor commands. + """ + + def __init__(self, name, filename, suppress_exceptions=False): + """ + Connect to the monitor socket, read the greeting message and issue the + qmp_capabilities command. Also make sure the json module is available. + + @param name: Monitor identifier (a string) + @param filename: Monitor socket filename + @raise MonitorConnectError: Raised if the connection fails and + suppress_exceptions is False + @raise MonitorProtocolError: Raised if the no QMP greeting message is + received and suppress_exceptions is False + @raise MonitorNotSupportedError: Raised if json isn't available and + suppress_exceptions is False + @note: Other exceptions may be raised if the qmp_capabilities command + fails. See cmd()'s docstring. + """ + try: + Monitor.__init__(self, name, filename) + + self.protocol = "qmp" + self._greeting = None + self._events = [] + + # Make sure json is available + try: + json + except NameError: + raise MonitorNotSupportedError("QMP requires the json module " + "(Python 2.6 and up)") + + # Read greeting message + end_time = time.time() + 20 + while time.time() < end_time: + for obj in self._read_objects(): + if "QMP" in obj: + self._greeting = obj + break + if self._greeting: + break + time.sleep(0.1) + else: + raise MonitorProtocolError("No QMP greeting message received") + + # Issue qmp_capabilities + self.cmd("qmp_capabilities") + + except MonitorError, e: + if suppress_exceptions: + logging.warn(e) + else: + raise + + + # Private methods + + def _build_cmd(self, cmd, args=None, id=None): + obj = {"execute": cmd} + if args is not None: + obj["arguments"] = args + if id is not None: + obj["id"] = id + return obj + + + def _read_objects(self, timeout=5): + """ + Read lines from the monitor and try to decode them. + Stop when all available lines have been successfully decoded, or when + timeout expires. If any decoded objects are asynchronous events, store + them in self._events. Return all decoded objects. + + @param timeout: Time to wait for all lines to decode successfully + @return: A list of objects + """ + if not self._data_available(): + return [] + s = "" + end_time = time.time() + timeout + while self._data_available(end_time - time.time()): + s += self._recvall() + # Make sure all lines are decodable + for line in s.splitlines(): + if line: + try: + json.loads(line) + except: + # Found an incomplete or broken line -- keep reading + break + else: + # All lines are OK -- stop reading + break + # Decode all decodable lines + objs = [] + for line in s.splitlines(): + try: + objs += [json.loads(line)] + except: + pass + # Keep track of asynchronous events + self._events += [obj for obj in objs if "event" in obj] + return objs + + + def _send(self, data): + """ + Send raw data without waiting for response. + + @param data: Data to send + @raise MonitorSocketError: Raised if a socket error occurs + """ + try: + self._socket.sendall(data) + except socket.error, e: + raise MonitorSocketError("Could not send data: %r" % data, e) + + + def _get_response(self, id=None, timeout=20): + """ + Read a response from the QMP monitor. + + @param id: If not None, look for a response with this id + @param timeout: Time duration to wait for response + @return: The response dict, or None if none was found + """ + end_time = time.time() + timeout + while self._data_available(end_time - time.time()): + for obj in self._read_objects(): + if isinstance(obj, dict): + if id is not None and obj.get("id") != id: + continue + if "return" in obj or "error" in obj: + return obj + + + # Public methods + + def cmd(self, cmd, args=None, timeout=20): + """ + Send a QMP monitor command and return the response. + + Note: an id is automatically assigned to the command and the response + is checked for the presence of the same id. + + @param cmd: Command to send + @param args: A dict containing command arguments, or None + @param timeout: Time duration to wait for response + @return: The response received + @raise MonitorLockError: Raised if the lock cannot be acquired + @raise MonitorSocketError: Raised if a socket error occurs + @raise MonitorProtocolError: Raised if no response is received + @raise QMPCmdError: Raised if the response is an error message + (the exception's args are (cmd, args, data) where data is the + error data) + """ + if not self._acquire_lock(20): + raise MonitorLockError("Could not acquire exclusive lock to send " + "QMP command '%s'" % cmd) + + try: + # Read any data that might be available + self._read_objects() + # Send command + id = virt_utils.generate_random_string(8) + self._send(json.dumps(self._build_cmd(cmd, args, id)) + "\n") + # Read response + r = self._get_response(id, timeout) + if r is None: + raise MonitorProtocolError("Received no response to QMP " + "command '%s', or received a " + "response with an incorrect id" + % cmd) + if "return" in r: + return r["return"] + if "error" in r: + raise QMPCmdError(cmd, args, r["error"]) + + finally: + self._lock.release() + + + def cmd_raw(self, data, timeout=20): + """ + Send a raw string to the QMP monitor and return the response. + Unlike cmd(), return the raw response dict without performing any + checks on it. + + @param data: The data to send + @param timeout: Time duration to wait for response + @return: The response received + @raise MonitorLockError: Raised if the lock cannot be acquired + @raise MonitorSocketError: Raised if a socket error occurs + @raise MonitorProtocolError: Raised if no response is received + """ + if not self._acquire_lock(20): + raise MonitorLockError("Could not acquire exclusive lock to send " + "data: %r" % data) + + try: + self._read_objects() + self._send(data) + r = self._get_response(None, timeout) + if r is None: + raise MonitorProtocolError("Received no response to data: %r" % + data) + return r + + finally: + self._lock.release() + + + def cmd_obj(self, obj, timeout=20): + """ + Transform a Python object to JSON, send the resulting string to the QMP + monitor, and return the response. + Unlike cmd(), return the raw response dict without performing any + checks on it. + + @param obj: The object to send + @param timeout: Time duration to wait for response + @return: The response received + @raise MonitorLockError: Raised if the lock cannot be acquired + @raise MonitorSocketError: Raised if a socket error occurs + @raise MonitorProtocolError: Raised if no response is received + """ + return self.cmd_raw(json.dumps(obj) + "\n") + + + def cmd_qmp(self, cmd, args=None, id=None, timeout=20): + """ + Build a QMP command from the passed arguments, send it to the monitor + and return the response. + Unlike cmd(), return the raw response dict without performing any + checks on it. + + @param cmd: Command to send + @param args: A dict containing command arguments, or None + @param id: An id for the command, or None + @param timeout: Time duration to wait for response + @return: The response received + @raise MonitorLockError: Raised if the lock cannot be acquired + @raise MonitorSocketError: Raised if a socket error occurs + @raise MonitorProtocolError: Raised if no response is received + """ + return self.cmd_obj(self._build_cmd(cmd, args, id), timeout) + + + def verify_responsive(self): + """ + Make sure the monitor is responsive by sending a command. + """ + self.cmd("query-status") + + + def get_events(self): + """ + Return a list of the asynchronous events received since the last + clear_events() call. + + @return: A list of events (the objects returned have an "event" key) + @raise MonitorLockError: Raised if the lock cannot be acquired + """ + if not self._acquire_lock(20): + raise MonitorLockError("Could not acquire exclusive lock to read " + "QMP events") + try: + self._read_objects() + return self._events[:] + finally: + self._lock.release() + + + def get_event(self, name): + """ + Look for an event with the given name in the list of events. + + @param name: The name of the event to look for (e.g. 'RESET') + @return: An event object or None if none is found + """ + for e in self.get_events(): + if e.get("event") == name: + return e + + + def clear_events(self): + """ + Clear the list of asynchronous events. + + @raise MonitorLockError: Raised if the lock cannot be acquired + """ + if not self._acquire_lock(20): + raise MonitorLockError("Could not acquire exclusive lock to clear " + "QMP event list") + self._events = [] + self._lock.release() + + + def get_greeting(self): + """ + Return QMP greeting message. + """ + return self._greeting + + + # Command wrappers + # Note: all of the following functions raise exceptions in a similar manner + # to cmd(). + + def quit(self): + """ + Send "quit" and return the response. + """ + return self.cmd("quit") + + + def info(self, what): + """ + Request info about something and return the response. + """ + return self.cmd("query-%s" % what) + + + def query(self, what): + """ + Alias for info. + """ + return self.info(what) + + + def screendump(self, filename): + """ + Request a screendump. + + @param filename: Location for the screendump + @return: The response to the command + """ + args = {"filename": filename} + return self.cmd("screendump", args) + + + def migrate(self, uri, full_copy=False, incremental_copy=False, wait=False): + """ + Migrate. + + @param uri: destination URI + @param full_copy: If true, migrate with full disk copy + @param incremental_copy: If true, migrate with incremental disk copy + @param wait: If true, wait for completion + @return: The response to the command + """ + args = {"uri": uri, + "blk": full_copy, + "inc": incremental_copy} + return self.cmd("migrate", args) + + + def migrate_set_speed(self, value): + """ + Set maximum speed (in bytes/sec) for migrations. + + @param value: Speed in bytes/sec + @return: The response to the command + """ + args = {"value": value} + return self.cmd("migrate_set_speed", args) diff --git a/client/virt/kvm_vm.py b/client/virt/kvm_vm.py new file mode 100755 index 0000000..82dae3e --- /dev/null +++ b/client/virt/kvm_vm.py @@ -0,0 +1,1500 @@ +#!/usr/bin/python +""" +Utility classes and functions to handle Virtual Machine creation using qemu. + +@copyright: 2008-2009 Red Hat Inc. +""" + +import time, os, logging, fcntl, re, commands, glob +from autotest_lib.client.common_lib import error +from autotest_lib.client.bin import utils +import virt_utils, virt_vm, kvm_monitor, aexpect + + +class VM: + """ + This class handles all basic VM operations. + """ + + def __init__(self, name, params, root_dir, address_cache, state=None): + """ + Initialize the object and set a few attributes. + + @param name: The name of the object + @param params: A dict containing VM params + (see method make_qemu_command for a full description) + @param root_dir: Base directory for relative filenames + @param address_cache: A dict that maps MAC addresses to IP addresses + @param state: If provided, use this as self.__dict__ + """ + if state: + self.__dict__ = state + else: + self.process = None + self.serial_console = None + self.redirs = {} + self.vnc_port = 5900 + self.monitors = [] + self.pci_assignable = None + self.netdev_id = [] + self.device_id = [] + self.uuid = None + + # Find a unique identifier for this VM + while True: + self.instance = (time.strftime("%Y%m%d-%H%M%S-") + + virt_utils.generate_random_string(4)) + if not glob.glob("/tmp/*%s" % self.instance): + break + + self.name = name + self.params = params + self.root_dir = root_dir + self.address_cache = address_cache + + + def clone(self, name=None, params=None, root_dir=None, address_cache=None, + copy_state=False): + """ + Return a clone of the VM object with optionally modified parameters. + The clone is initially not alive and needs to be started using create(). + Any parameters not passed to this function are copied from the source + VM. + + @param name: Optional new VM name + @param params: Optional new VM creation parameters + @param root_dir: Optional new base directory for relative filenames + @param address_cache: A dict that maps MAC addresses to IP addresses + @param copy_state: If True, copy the original VM's state to the clone. + Mainly useful for make_qemu_command(). + """ + if name is None: + name = self.name + if params is None: + params = self.params.copy() + if root_dir is None: + root_dir = self.root_dir + if address_cache is None: + address_cache = self.address_cache + if copy_state: + state = self.__dict__.copy() + else: + state = None + return VM(name, params, root_dir, address_cache, state) + + + def make_qemu_command(self, name=None, params=None, root_dir=None): + """ + Generate a qemu command line. All parameters are optional. If a + parameter is not supplied, the corresponding value stored in the + class attributes is used. + + @param name: The name of the object + @param params: A dict containing VM params + @param root_dir: Base directory for relative filenames + + @note: The params dict should contain: + mem -- memory size in MBs + cdrom -- ISO filename to use with the qemu -cdrom parameter + extra_params -- a string to append to the qemu command + shell_port -- port of the remote shell daemon on the guest + (SSH, Telnet or the home-made Remote Shell Server) + shell_client -- client program to use for connecting to the + remote shell daemon on the guest (ssh, telnet or nc) + x11_display -- if specified, the DISPLAY environment variable + will be be set to this value for the qemu process (useful for + SDL rendering) + images -- a list of image object names, separated by spaces + nics -- a list of NIC object names, separated by spaces + + For each image in images: + drive_format -- string to pass as 'if' parameter for this + image (e.g. ide, scsi) + image_snapshot -- if yes, pass 'snapshot=on' to qemu for + this image + image_boot -- if yes, pass 'boot=on' to qemu for this image + In addition, all parameters required by get_image_filename. + + For each NIC in nics: + nic_model -- string to pass as 'model' parameter for this + NIC (e.g. e1000) + """ + # Helper function for command line option wrappers + def has_option(help, option): + return bool(re.search(r"^-%s(\s|$)" % option, help, re.MULTILINE)) + + # Wrappers for all supported qemu command line parameters. + # This is meant to allow support for multiple qemu versions. + # Each of these functions receives the output of 'qemu -help' as a + # parameter, and should add the requested command line option + # accordingly. + + def add_name(help, name): + return " -name '%s'" % name + + def add_human_monitor(help, filename): + return " -monitor unix:'%s',server,nowait" % filename + + def add_qmp_monitor(help, filename): + return " -qmp unix:'%s',server,nowait" % filename + + def add_serial(help, filename): + return " -serial unix:'%s',server,nowait" % filename + + def add_mem(help, mem): + return " -m %s" % mem + + def add_smp(help, smp): + return " -smp %s" % smp + + def add_cdrom(help, filename, index=None): + if has_option(help, "drive"): + cmd = " -drive file='%s',media=cdrom" % filename + if index is not None: cmd += ",index=%s" % index + return cmd + else: + return " -cdrom '%s'" % filename + + def add_drive(help, filename, index=None, format=None, cache=None, + werror=None, serial=None, snapshot=False, boot=False): + cmd = " -drive file='%s'" % filename + if index is not None: + cmd += ",index=%s" % index + if format: + cmd += ",if=%s" % format + if cache: + cmd += ",cache=%s" % cache + if werror: + cmd += ",werror=%s" % werror + if serial: + cmd += ",serial='%s'" % serial + if snapshot: + cmd += ",snapshot=on" + if boot: + cmd += ",boot=on" + return cmd + + def add_nic(help, vlan, model=None, mac=None, device_id=None, netdev_id=None, + nic_extra_params=None): + if has_option(help, "netdev"): + netdev_vlan_str = ",netdev=%s" % netdev_id + else: + netdev_vlan_str = ",vlan=%d" % vlan + if has_option(help, "device"): + if not model: + model = "rtl8139" + elif model == "virtio": + model = "virtio-net-pci" + cmd = " -device %s" % model + netdev_vlan_str + if mac: + cmd += ",mac='%s'" % mac + if nic_extra_params: + cmd += ",%s" % nic_extra_params + else: + cmd = " -net nic" + netdev_vlan_str + if model: + cmd += ",model=%s" % model + if mac: + cmd += ",macaddr='%s'" % mac + if device_id: + cmd += ",id='%s'" % device_id + return cmd + + def add_net(help, vlan, mode, ifname=None, script=None, + downscript=None, tftp=None, bootfile=None, hostfwd=[], + netdev_id=None, netdev_extra_params=None): + if has_option(help, "netdev"): + cmd = " -netdev %s,id=%s" % (mode, netdev_id) + if netdev_extra_params: + cmd += ",%s" % netdev_extra_params + else: + cmd = " -net %s,vlan=%d" % (mode, vlan) + if mode == "tap": + if ifname: cmd += ",ifname='%s'" % ifname + if script: cmd += ",script='%s'" % script + cmd += ",downscript='%s'" % (downscript or "no") + elif mode == "user": + if tftp and "[,tftp=" in help: + cmd += ",tftp='%s'" % tftp + if bootfile and "[,bootfile=" in help: + cmd += ",bootfile='%s'" % bootfile + if "[,hostfwd=" in help: + for host_port, guest_port in hostfwd: + cmd += ",hostfwd=tcp::%s-:%s" % (host_port, guest_port) + return cmd + + def add_floppy(help, filename): + return " -fda '%s'" % filename + + def add_tftp(help, filename): + # If the new syntax is supported, don't add -tftp + if "[,tftp=" in help: + return "" + else: + return " -tftp '%s'" % filename + + def add_bootp(help, filename): + # If the new syntax is supported, don't add -bootp + if "[,bootfile=" in help: + return "" + else: + return " -bootp '%s'" % filename + + def add_tcp_redir(help, host_port, guest_port): + # If the new syntax is supported, don't add -redir + if "[,hostfwd=" in help: + return "" + else: + return " -redir tcp:%s::%s" % (host_port, guest_port) + + def add_vnc(help, vnc_port): + return " -vnc :%d" % (vnc_port - 5900) + + def add_sdl(help): + if has_option(help, "sdl"): + return " -sdl" + else: + return "" + + def add_nographic(help): + return " -nographic" + + def add_uuid(help, uuid): + return " -uuid '%s'" % uuid + + def add_pcidevice(help, host): + return " -pcidevice host='%s'" % host + + def add_kernel(help, filename): + return " -kernel '%s'" % filename + + def add_initrd(help, filename): + return " -initrd '%s'" % filename + + def add_kernel_cmdline(help, cmdline): + return " -append %s" % cmdline + + def add_testdev(help, filename): + return (" -chardev file,id=testlog,path=%s" + " -device testdev,chardev=testlog" % filename) + + def add_no_hpet(help): + if has_option(help, "no-hpet"): + return " -no-hpet" + else: + return "" + + # End of command line option wrappers + + if name is None: + name = self.name + if params is None: + params = self.params + if root_dir is None: + root_dir = self.root_dir + + # Clone this VM using the new params + vm = self.clone(name, params, root_dir, copy_state=True) + + qemu_binary = virt_utils.get_path(root_dir, params.get("qemu_binary", + "qemu")) + # Get the output of 'qemu -help' (log a message in case this call never + # returns or causes some other kind of trouble) + logging.debug("Getting output of 'qemu -help'") + help = commands.getoutput("%s -help" % qemu_binary) + + # Start constructing the qemu command + qemu_cmd = "" + # Set the X11 display parameter if requested + if params.get("x11_display"): + qemu_cmd += "DISPLAY=%s " % params.get("x11_display") + # Add the qemu binary + qemu_cmd += qemu_binary + # Add the VM's name + qemu_cmd += add_name(help, name) + # Add monitors + for monitor_name in params.objects("monitors"): + monitor_params = params.object_params(monitor_name) + monitor_filename = vm.get_monitor_filename(monitor_name) + if monitor_params.get("monitor_type") == "qmp": + qemu_cmd += add_qmp_monitor(help, monitor_filename) + else: + qemu_cmd += add_human_monitor(help, monitor_filename) + + # Add serial console redirection + qemu_cmd += add_serial(help, vm.get_serial_console_filename()) + + for image_name in params.objects("images"): + image_params = params.object_params(image_name) + if image_params.get("boot_drive") == "no": + continue + qemu_cmd += add_drive(help, + virt_vm.get_image_filename(image_params, root_dir), + image_params.get("drive_index"), + image_params.get("drive_format"), + image_params.get("drive_cache"), + image_params.get("drive_werror"), + image_params.get("drive_serial"), + image_params.get("image_snapshot") == "yes", + image_params.get("image_boot") == "yes") + + redirs = [] + for redir_name in params.objects("redirs"): + redir_params = params.object_params(redir_name) + guest_port = int(redir_params.get("guest_port")) + host_port = vm.redirs.get(guest_port) + redirs += [(host_port, guest_port)] + + vlan = 0 + for nic_name in params.objects("nics"): + nic_params = params.object_params(nic_name) + try: + netdev_id = vm.netdev_id[vlan] + device_id = vm.device_id[vlan] + except IndexError: + netdev_id = None + # Handle the '-net nic' part + try: + mac = vm.get_mac_address(vlan) + except virt_vm.VMAddressError: + mac = None + qemu_cmd += add_nic(help, vlan, nic_params.get("nic_model"), mac, + device_id, netdev_id, nic_params.get("nic_extra_params")) + # Handle the '-net tap' or '-net user' or '-netdev' part + script = nic_params.get("nic_script") + downscript = nic_params.get("nic_downscript") + tftp = nic_params.get("tftp") + if script: + script = virt_utils.get_path(root_dir, script) + if downscript: + downscript = virt_utils.get_path(root_dir, downscript) + if tftp: + tftp = virt_utils.get_path(root_dir, tftp) + qemu_cmd += add_net(help, vlan, nic_params.get("nic_mode", "user"), + vm.get_ifname(vlan), + script, downscript, tftp, + nic_params.get("bootp"), redirs, netdev_id, + nic_params.get("netdev_extra_params")) + # Proceed to next NIC + vlan += 1 + + mem = params.get("mem") + if mem: + qemu_cmd += add_mem(help, mem) + + smp = params.get("smp") + if smp: + qemu_cmd += add_smp(help, smp) + + for cdrom in params.objects("cdroms"): + cdrom_params = params.object_params(cdrom) + iso = cdrom_params.get("cdrom") + if iso: + qemu_cmd += add_cdrom(help, virt_utils.get_path(root_dir, iso), + cdrom_params.get("drive_index")) + + # We may want to add {floppy_otps} parameter for -fda + # {fat:floppy:}/path/. However vvfat is not usually recommended. + floppy = params.get("floppy") + if floppy: + floppy = virt_utils.get_path(root_dir, floppy) + qemu_cmd += add_floppy(help, floppy) + + tftp = params.get("tftp") + if tftp: + tftp = virt_utils.get_path(root_dir, tftp) + qemu_cmd += add_tftp(help, tftp) + + bootp = params.get("bootp") + if bootp: + qemu_cmd += add_bootp(help, bootp) + + kernel = params.get("kernel") + if kernel: + kernel = virt_utils.get_path(root_dir, kernel) + qemu_cmd += add_kernel(help, kernel) + + kernel_cmdline = params.get("kernel_cmdline") + if kernel_cmdline: + qemu_cmd += add_kernel_cmdline(help, kernel_cmdline) + + initrd = params.get("initrd") + if initrd: + initrd = virt_utils.get_path(root_dir, initrd) + qemu_cmd += add_initrd(help, initrd) + + for host_port, guest_port in redirs: + qemu_cmd += add_tcp_redir(help, host_port, guest_port) + + if params.get("display") == "vnc": + qemu_cmd += add_vnc(help, vm.vnc_port) + elif params.get("display") == "sdl": + qemu_cmd += add_sdl(help) + elif params.get("display") == "nographic": + qemu_cmd += add_nographic(help) + + if params.get("uuid") == "random": + qemu_cmd += add_uuid(help, vm.uuid) + elif params.get("uuid"): + qemu_cmd += add_uuid(help, params.get("uuid")) + + if params.get("testdev") == "yes": + qemu_cmd += add_testdev(help, vm.get_testlog_filename()) + + if params.get("disable_hpet") == "yes": + qemu_cmd += add_no_hpet(help) + + # If the PCI assignment step went OK, add each one of the PCI assigned + # devices to the qemu command line. + if vm.pci_assignable: + for pci_id in vm.pa_pci_ids: + qemu_cmd += add_pcidevice(help, pci_id) + + extra_params = params.get("extra_params") + if extra_params: + qemu_cmd += " %s" % extra_params + + return qemu_cmd + + + @error.context_aware + def create(self, name=None, params=None, root_dir=None, timeout=5.0, + migration_mode=None, mac_source=None): + """ + Start the VM by running a qemu command. + All parameters are optional. If name, params or root_dir are not + supplied, the respective values stored as class attributes are used. + + @param name: The name of the object + @param params: A dict containing VM params + @param root_dir: Base directory for relative filenames + @param migration_mode: If supplied, start VM for incoming migration + using this protocol (either 'tcp', 'unix' or 'exec') + @param migration_exec_cmd: Command to embed in '-incoming "exec: ..."' + (e.g. 'gzip -c -d filename') if migration_mode is 'exec' + @param mac_source: A VM object from which to copy MAC addresses. If not + specified, new addresses will be generated. + + @raise VMCreateError: If qemu terminates unexpectedly + @raise VMKVMInitError: If KVM initialization fails + @raise VMHugePageError: If hugepage initialization fails + @raise VMImageMissingError: If a CD image is missing + @raise VMHashMismatchError: If a CD image hash has doesn't match the + expected hash + @raise VMBadPATypeError: If an unsupported PCI assignment type is + requested + @raise VMPAError: If no PCI assignable devices could be assigned + """ + error.context("creating '%s'" % self.name) + self.destroy(free_mac_addresses=False) + + if name is not None: + self.name = name + if params is not None: + self.params = params + if root_dir is not None: + self.root_dir = root_dir + name = self.name + params = self.params + root_dir = self.root_dir + + # Verify the md5sum of the ISO images + for cdrom in params.objects("cdroms"): + cdrom_params = params.object_params(cdrom) + iso = cdrom_params.get("cdrom") + if iso: + iso = virt_utils.get_path(root_dir, iso) + if not os.path.exists(iso): + raise virt_vm.VMImageMissingError(iso) + compare = False + if cdrom_params.get("md5sum_1m"): + logging.debug("Comparing expected MD5 sum with MD5 sum of " + "first MB of ISO file...") + actual_hash = utils.hash_file(iso, 1048576, method="md5") + expected_hash = cdrom_params.get("md5sum_1m") + compare = True + elif cdrom_params.get("md5sum"): + logging.debug("Comparing expected MD5 sum with MD5 sum of " + "ISO file...") + actual_hash = utils.hash_file(iso, method="md5") + expected_hash = cdrom_params.get("md5sum") + compare = True + elif cdrom_params.get("sha1sum"): + logging.debug("Comparing expected SHA1 sum with SHA1 sum " + "of ISO file...") + actual_hash = utils.hash_file(iso, method="sha1") + expected_hash = cdrom_params.get("sha1sum") + compare = True + if compare: + if actual_hash == expected_hash: + logging.debug("Hashes match") + else: + raise virt_vm.VMHashMismatchError(actual_hash, + expected_hash) + + # Make sure the following code is not executed by more than one thread + # at the same time + lockfile = open("/tmp/kvm-autotest-vm-create.lock", "w+") + fcntl.lockf(lockfile, fcntl.LOCK_EX) + + try: + # Handle port redirections + redir_names = params.objects("redirs") + host_ports = virt_utils.find_free_ports(5000, 6000, len(redir_names)) + self.redirs = {} + for i in range(len(redir_names)): + redir_params = params.object_params(redir_names[i]) + guest_port = int(redir_params.get("guest_port")) + self.redirs[guest_port] = host_ports[i] + + # Generate netdev/device IDs for all NICs + self.netdev_id = [] + self.device_id = [] + for nic in params.objects("nics"): + self.netdev_id.append(virt_utils.generate_random_id()) + self.device_id.append(virt_utils.generate_random_id()) + + # Find available VNC port, if needed + if params.get("display") == "vnc": + self.vnc_port = virt_utils.find_free_port(5900, 6100) + + # Find random UUID if specified 'uuid = random' in config file + if params.get("uuid") == "random": + f = open("/proc/sys/kernel/random/uuid") + self.uuid = f.read().strip() + f.close() + + # Generate or copy MAC addresses for all NICs + num_nics = len(params.objects("nics")) + for vlan in range(num_nics): + nic_name = params.objects("nics")[vlan] + nic_params = params.object_params(nic_name) + mac = (nic_params.get("nic_mac") or + mac_source and mac_source.get_mac_address(vlan)) + if mac: + virt_utils.set_mac_address(self.instance, vlan, mac) + else: + virt_utils.generate_mac_address(self.instance, vlan) + + # Assign a PCI assignable device + self.pci_assignable = None + pa_type = params.get("pci_assignable") + if pa_type and pa_type != "no": + pa_devices_requested = params.get("devices_requested") + + # Virtual Functions (VF) assignable devices + if pa_type == "vf": + self.pci_assignable = virt_utils.PciAssignable( + type=pa_type, + driver=params.get("driver"), + driver_option=params.get("driver_option"), + devices_requested=pa_devices_requested) + # Physical NIC (PF) assignable devices + elif pa_type == "pf": + self.pci_assignable = virt_utils.PciAssignable( + type=pa_type, + names=params.get("device_names"), + devices_requested=pa_devices_requested) + # Working with both VF and PF + elif pa_type == "mixed": + self.pci_assignable = virt_utils.PciAssignable( + type=pa_type, + driver=params.get("driver"), + driver_option=params.get("driver_option"), + names=params.get("device_names"), + devices_requested=pa_devices_requested) + else: + raise virt_vm.VMBadPATypeError(pa_type) + + self.pa_pci_ids = self.pci_assignable.request_devs() + + if self.pa_pci_ids: + logging.debug("Successfuly assigned devices: %s", + self.pa_pci_ids) + else: + raise virt_vm.VMPAError(pa_type) + + # Make qemu command + qemu_command = self.make_qemu_command() + + # Add migration parameters if required + if migration_mode == "tcp": + self.migration_port = virt_utils.find_free_port(5200, 6000) + qemu_command += " -incoming tcp:0:%d" % self.migration_port + elif migration_mode == "unix": + self.migration_file = "/tmp/migration-unix-%s" % self.instance + qemu_command += " -incoming unix:%s" % self.migration_file + elif migration_mode == "exec": + self.migration_port = virt_utils.find_free_port(5200, 6000) + qemu_command += (' -incoming "exec:nc -l %s"' % + self.migration_port) + + logging.info("Running qemu command:\n%s", qemu_command) + self.process = aexpect.run_bg(qemu_command, None, + logging.info, "(qemu) ") + + # Make sure the process was started successfully + if not self.process.is_alive(): + e = virt_vm.VMCreateError(qemu_command, + self.process.get_status(), + self.process.get_output()) + self.destroy() + raise e + + # Establish monitor connections + self.monitors = [] + for monitor_name in params.objects("monitors"): + monitor_params = params.object_params(monitor_name) + # Wait for monitor connection to succeed + end_time = time.time() + timeout + while time.time() < end_time: + try: + if monitor_params.get("monitor_type") == "qmp": + # Add a QMP monitor + monitor = kvm_monitor.QMPMonitor( + monitor_name, + self.get_monitor_filename(monitor_name)) + else: + # Add a "human" monitor + monitor = kvm_monitor.HumanMonitor( + monitor_name, + self.get_monitor_filename(monitor_name)) + monitor.verify_responsive() + break + except kvm_monitor.MonitorError, e: + logging.warn(e) + time.sleep(1) + else: + self.destroy() + raise e + # Add this monitor to the list + self.monitors += [monitor] + + # Get the output so far, to see if we have any problems with + # KVM modules or with hugepage setup. + output = self.process.get_output() + + if re.search("Could not initialize KVM", output, re.IGNORECASE): + e = virt_vm.VMKVMInitError(qemu_command, self.process.get_output()) + self.destroy() + raise e + + if "alloc_mem_area" in output: + e = virt_vm.VMHugePageError(qemu_command, self.process.get_output()) + self.destroy() + raise e + + logging.debug("VM appears to be alive with PID %s", self.get_pid()) + + # Establish a session with the serial console -- requires a version + # of netcat that supports -U + self.serial_console = aexpect.ShellSession( + "nc -U %s" % self.get_serial_console_filename(), + auto_close=False, + output_func=virt_utils.log_line, + output_params=("serial-%s.log" % name,)) + + finally: + fcntl.lockf(lockfile, fcntl.LOCK_UN) + lockfile.close() + + + def destroy(self, gracefully=True, free_mac_addresses=True): + """ + Destroy the VM. + + If gracefully is True, first attempt to shutdown the VM with a shell + command. Then, attempt to destroy the VM via the monitor with a 'quit' + command. If that fails, send SIGKILL to the qemu process. + + @param gracefully: If True, an attempt will be made to end the VM + using a shell command before trying to end the qemu process + with a 'quit' or a kill signal. + @param free_mac_addresses: If True, the MAC addresses used by the VM + will be freed. + """ + try: + # Is it already dead? + if self.is_dead(): + return + + logging.debug("Destroying VM with PID %s...", self.get_pid()) + + if gracefully and self.params.get("shutdown_command"): + # Try to destroy with shell command + logging.debug("Trying to shutdown VM with shell command...") + try: + session = self.login() + except (virt_utils.LoginError, virt_vm.VMError), e: + logging.debug(e) + else: + try: + # Send the shutdown command + session.sendline(self.params.get("shutdown_command")) + logging.debug("Shutdown command sent; waiting for VM " + "to go down...") + if virt_utils.wait_for(self.is_dead, 60, 1, 1): + logging.debug("VM is down") + return + finally: + session.close() + + if self.monitor: + # Try to destroy with a monitor command + logging.debug("Trying to kill VM with monitor command...") + try: + self.monitor.quit() + except kvm_monitor.MonitorError, e: + logging.warn(e) + else: + # Wait for the VM to be really dead + if virt_utils.wait_for(self.is_dead, 5, 0.5, 0.5): + logging.debug("VM is down") + return + + # If the VM isn't dead yet... + logging.debug("Cannot quit normally; sending a kill to close the " + "deal...") + virt_utils.kill_process_tree(self.process.get_pid(), 9) + # Wait for the VM to be really dead + if virt_utils.wait_for(self.is_dead, 5, 0.5, 0.5): + logging.debug("VM is down") + return + + logging.error("Process %s is a zombie!", self.process.get_pid()) + + finally: + self.monitors = [] + if self.pci_assignable: + self.pci_assignable.release_devs() + if self.process: + self.process.close() + if self.serial_console: + self.serial_console.close() + for f in ([self.get_testlog_filename(), + self.get_serial_console_filename()] + + self.get_monitor_filenames()): + try: + os.unlink(f) + except OSError: + pass + if hasattr(self, "migration_file"): + try: + os.unlink(self.migration_file) + except OSError: + pass + if free_mac_addresses: + num_nics = len(self.params.objects("nics")) + for vlan in range(num_nics): + self.free_mac_address(vlan) + + + @property + def monitor(self): + """ + Return the main monitor object, selected by the parameter main_monitor. + If main_monitor isn't defined, return the first monitor. + If no monitors exist, or if main_monitor refers to a nonexistent + monitor, return None. + """ + for m in self.monitors: + if m.name == self.params.get("main_monitor"): + return m + if self.monitors and not self.params.get("main_monitor"): + return self.monitors[0] + + + def verify_alive(self): + """ + Make sure the VM is alive and that the main monitor is responsive. + + @raise VMDeadError: If the VM is dead + @raise: Various monitor exceptions if the monitor is unresponsive + """ + if self.is_dead(): + raise virt_vm.VMDeadError(self.process.get_status(), + self.process.get_output()) + if self.monitors: + self.monitor.verify_responsive() + + + def is_alive(self): + """ + Return True if the VM is alive and its monitor is responsive. + """ + return not self.is_dead() and (not self.monitors or + self.monitor.is_responsive()) + + + def is_dead(self): + """ + Return True if the qemu process is dead. + """ + return not self.process or not self.process.is_alive() + + + def get_params(self): + """ + Return the VM's params dict. Most modified params take effect only + upon VM.create(). + """ + return self.params + + + def get_monitor_filename(self, monitor_name): + """ + Return the filename corresponding to a given monitor name. + """ + return "/tmp/monitor-%s-%s" % (monitor_name, self.instance) + + + def get_monitor_filenames(self): + """ + Return a list of all monitor filenames (as specified in the VM's + params). + """ + return [self.get_monitor_filename(m) for m in + self.params.objects("monitors")] + + + def get_serial_console_filename(self): + """ + Return the serial console filename. + """ + return "/tmp/serial-%s" % self.instance + + + def get_testlog_filename(self): + """ + Return the testlog filename. + """ + return "/tmp/testlog-%s" % self.instance + + + def get_address(self, index=0): + """ + Return the address of a NIC of the guest, in host space. + + If port redirection is used, return 'localhost' (the NIC has no IP + address of its own). Otherwise return the NIC's IP address. + + @param index: Index of the NIC whose address is requested. + @raise VMMACAddressMissingError: If no MAC address is defined for the + requested NIC + @raise VMIPAddressMissingError: If no IP address is found for the the + NIC's MAC address + @raise VMAddressVerificationError: If the MAC-IP address mapping cannot + be verified (using arping) + """ + nics = self.params.objects("nics") + nic_name = nics[index] + nic_params = self.params.object_params(nic_name) + if nic_params.get("nic_mode") == "tap": + mac = self.get_mac_address(index).lower() + # Get the IP address from the cache + ip = self.address_cache.get(mac) + if not ip: + raise virt_vm.VMIPAddressMissingError(mac) + # Make sure the IP address is assigned to this guest + macs = [self.get_mac_address(i) for i in range(len(nics))] + if not virt_utils.verify_ip_address_ownership(ip, macs): + raise virt_vm.VMAddressVerificationError(mac, ip) + return ip + else: + return "localhost" + + + def get_port(self, port, nic_index=0): + """ + Return the port in host space corresponding to port in guest space. + + @param port: Port number in host space. + @param nic_index: Index of the NIC. + @return: If port redirection is used, return the host port redirected + to guest port port. Otherwise return port. + @raise VMPortNotRedirectedError: If an unredirected port is requested + in user mode + """ + nic_name = self.params.objects("nics")[nic_index] + nic_params = self.params.object_params(nic_name) + if nic_params.get("nic_mode") == "tap": + return port + else: + try: + return self.redirs[port] + except KeyError: + raise virt_vm.VMPortNotRedirectedError(port) + + + def get_peer(self, netid): + """ + Return the peer of netdev or network deivce. + + @param netid: id of netdev or device + @return: id of the peer device otherwise None + """ + network_info = self.monitor.info("network") + try: + return re.findall("%s:.*peer=(.*)" % netid, network_info)[0] + except IndexError: + return None + + + def get_ifname(self, nic_index=0): + """ + Return the ifname of a tap device associated with a NIC. + + @param nic_index: Index of the NIC + """ + nics = self.params.objects("nics") + nic_name = nics[nic_index] + nic_params = self.params.object_params(nic_name) + if nic_params.get("nic_ifname"): + return nic_params.get("nic_ifname") + else: + return "t%d-%s" % (nic_index, self.instance[-11:]) + + + def get_mac_address(self, nic_index=0): + """ + Return the MAC address of a NIC. + + @param nic_index: Index of the NIC + @raise VMMACAddressMissingError: If no MAC address is defined for the + requested NIC + """ + nic_name = self.params.objects("nics")[nic_index] + nic_params = self.params.object_params(nic_name) + mac = (nic_params.get("nic_mac") or + virt_utils.get_mac_address(self.instance, nic_index)) + if not mac: + raise virt_vm.VMMACAddressMissingError(nic_index) + return mac + + + def free_mac_address(self, nic_index=0): + """ + Free a NIC's MAC address. + + @param nic_index: Index of the NIC + """ + virt_utils.free_mac_address(self.instance, nic_index) + + + def get_pid(self): + """ + Return the VM's PID. If the VM is dead return None. + + @note: This works under the assumption that self.process.get_pid() + returns the PID of the parent shell process. + """ + try: + children = commands.getoutput("ps --ppid=%d -o pid=" % + self.process.get_pid()).split() + return int(children[0]) + except (TypeError, IndexError, ValueError): + return None + + + def get_shell_pid(self): + """ + Return the PID of the parent shell process. + + @note: This works under the assumption that self.process.get_pid() + returns the PID of the parent shell process. + """ + return self.process.get_pid() + + + def get_shared_meminfo(self): + """ + Returns the VM's shared memory information. + + @return: Shared memory used by VM (MB) + """ + if self.is_dead(): + logging.error("Could not get shared memory info from dead VM.") + return None + + filename = "/proc/%d/statm" % self.get_pid() + shm = int(open(filename).read().split()[2]) + # statm stores informations in pages, translate it to MB + return shm * 4.0 / 1024 + + + @error.context_aware + def login(self, nic_index=0, timeout=10): + """ + Log into the guest via SSH/Telnet/Netcat. + If timeout expires while waiting for output from the guest (e.g. a + password prompt or a shell prompt) -- fail. + + @param nic_index: The index of the NIC to connect to. + @param timeout: Time (seconds) before giving up logging into the + guest. + @return: A ShellSession object. + """ + error.context("logging into '%s'" % self.name) + username = self.params.get("username", "") + password = self.params.get("password", "") + prompt = self.params.get("shell_prompt", "[\#\$]") + linesep = eval("'%s'" % self.params.get("shell_linesep", r"\n")) + client = self.params.get("shell_client") + address = self.get_address(nic_index) + port = self.get_port(int(self.params.get("shell_port"))) + log_filename = ("session-%s-%s.log" % + (self.name, virt_utils.generate_random_string(4))) + session = virt_utils.remote_login(client, address, port, username, + password, prompt, linesep, + log_filename, timeout) + session.set_status_test_command(self.params.get("status_test_command", + "")) + return session + + + def remote_login(self, nic_index=0, timeout=10): + """ + Alias for login() for backward compatibility. + """ + return self.login(nic_index, timeout) + + + def wait_for_login(self, nic_index=0, timeout=240, internal_timeout=10): + """ + Make multiple attempts to log into the guest via SSH/Telnet/Netcat. + + @param nic_index: The index of the NIC to connect to. + @param timeout: Time (seconds) to keep trying to log in. + @param internal_timeout: Timeout to pass to login(). + @return: A ShellSession object. + """ + logging.debug("Attempting to log into '%s' (timeout %ds)", self.name, + timeout) + end_time = time.time() + timeout + while time.time() < end_time: + try: + return self.login(nic_index, internal_timeout) + except (virt_utils.LoginError, virt_vm.VMError), e: + logging.debug(e) + time.sleep(2) + # Timeout expired; try one more time but don't catch exceptions + return self.login(nic_index, internal_timeout) + + + @error.context_aware + def copy_files_to(self, host_path, guest_path, nic_index=0, verbose=False, + timeout=600): + """ + Transfer files to the remote host(guest). + + @param host_path: Host path + @param guest_path: Guest path + @param nic_index: The index of the NIC to connect to. + @param verbose: If True, log some stats using logging.debug (RSS only) + @param timeout: Time (seconds) before giving up on doing the remote + copy. + """ + error.context("sending file(s) to '%s'" % self.name) + username = self.params.get("username", "") + password = self.params.get("password", "") + client = self.params.get("file_transfer_client") + address = self.get_address(nic_index) + port = self.get_port(int(self.params.get("file_transfer_port"))) + log_filename = ("transfer-%s-to-%s-%s.log" % + (self.name, address, + virt_utils.generate_random_string(4))) + virt_utils.copy_files_to(address, client, username, password, port, + host_path, guest_path, log_filename, verbose, + timeout) + + + @error.context_aware + def copy_files_from(self, guest_path, host_path, nic_index=0, + verbose=False, timeout=600): + """ + Transfer files from the guest. + + @param host_path: Guest path + @param guest_path: Host path + @param nic_index: The index of the NIC to connect to. + @param verbose: If True, log some stats using logging.debug (RSS only) + @param timeout: Time (seconds) before giving up on doing the remote + copy. + """ + error.context("receiving file(s) from '%s'" % self.name) + username = self.params.get("username", "") + password = self.params.get("password", "") + client = self.params.get("file_transfer_client") + address = self.get_address(nic_index) + port = self.get_port(int(self.params.get("file_transfer_port"))) + log_filename = ("transfer-%s-from-%s-%s.log" % + (self.name, address, + virt_utils.generate_random_string(4))) + virt_utils.copy_files_from(address, client, username, password, port, + guest_path, host_path, log_filename, + verbose, timeout) + + + @error.context_aware + def serial_login(self, timeout=10): + """ + Log into the guest via the serial console. + If timeout expires while waiting for output from the guest (e.g. a + password prompt or a shell prompt) -- fail. + + @param timeout: Time (seconds) before giving up logging into the guest. + @return: ShellSession object on success and None on failure. + """ + error.context("logging into '%s' via serial console" % self.name) + username = self.params.get("username", "") + password = self.params.get("password", "") + prompt = self.params.get("shell_prompt", "[\#\$]") + linesep = eval("'%s'" % self.params.get("shell_linesep", r"\n")) + status_test_command = self.params.get("status_test_command", "") + + self.serial_console.set_linesep(linesep) + self.serial_console.set_status_test_command(status_test_command) + + # Try to get a login prompt + self.serial_console.sendline() + + virt_utils._remote_login(self.serial_console, username, password, + prompt, timeout) + return self.serial_console + + + def wait_for_serial_login(self, timeout=240, internal_timeout=10): + """ + Make multiple attempts to log into the guest via serial console. + + @param timeout: Time (seconds) to keep trying to log in. + @param internal_timeout: Timeout to pass to serial_login(). + @return: A ShellSession object. + """ + logging.debug("Attempting to log into '%s' via serial console " + "(timeout %ds)", self.name, timeout) + end_time = time.time() + timeout + while time.time() < end_time: + try: + return self.serial_login(internal_timeout) + except virt_utils.LoginError, e: + logging.debug(e) + time.sleep(2) + # Timeout expired; try one more time but don't catch exceptions + return self.serial_login(internal_timeout) + + + @error.context_aware + def migrate(self, timeout=3600, protocol="tcp", cancel_delay=None, + offline=False, stable_check=False, clean=True, + save_path="/tmp", dest_host="localhost", remote_port=None): + """ + Migrate the VM. + + If the migration is local, the VM object's state is switched with that + of the destination VM. Otherwise, the state is switched with that of + a dead VM (returned by self.clone()). + + @param timeout: Time to wait for migration to complete. + @param protocol: Migration protocol ('tcp', 'unix' or 'exec'). + @param cancel_delay: If provided, specifies a time duration after which + migration will be canceled. Used for testing migrate_cancel. + @param offline: If True, pause the source VM before migration. + @param stable_check: If True, compare the VM's state after migration to + its state before migration and raise an exception if they + differ. + @param clean: If True, delete the saved state files (relevant only if + stable_check is also True). + @save_path: The path for state files. + @param dest_host: Destination host (defaults to 'localhost'). + @param remote_port: Port to use for remote migration. + """ + error.base_context("migrating '%s'" % self.name) + + def mig_finished(): + o = self.monitor.info("migrate") + if isinstance(o, str): + return "status: active" not in o + else: + return o.get("status") != "active" + + def mig_succeeded(): + o = self.monitor.info("migrate") + if isinstance(o, str): + return "status: completed" in o + else: + return o.get("status") == "completed" + + def mig_failed(): + o = self.monitor.info("migrate") + if isinstance(o, str): + return "status: failed" in o + else: + return o.get("status") == "failed" + + def mig_cancelled(): + o = self.monitor.info("migrate") + if isinstance(o, str): + return ("Migration status: cancelled" in o or + "Migration status: canceled" in o) + else: + return (o.get("status") == "cancelled" or + o.get("status") == "canceled") + + def wait_for_migration(): + if not virt_utils.wait_for(mig_finished, timeout, 2, 2, + "Waiting for migration to complete"): + raise virt_vm.VMMigrateTimeoutError("Timeout expired while waiting " + "for migration to finish") + + local = dest_host == "localhost" + + clone = self.clone() + if local: + error.context("creating destination VM") + if stable_check: + # Pause the dest vm after creation + extra_params = clone.params.get("extra_params", "") + " -S" + clone.params["extra_params"] = extra_params + clone.create(migration_mode=protocol, mac_source=self) + error.context() + + try: + if protocol == "tcp": + if local: + uri = "tcp:localhost:%d" % clone.migration_port + else: + uri = "tcp:%s:%d" % (dest_host, remote_port) + elif protocol == "unix": + uri = "unix:%s" % clone.migration_file + elif protocol == "exec": + uri = '"exec:nc localhost %s"' % clone.migration_port + + if offline: + self.monitor.cmd("stop") + + logging.info("Migrating to %s", uri) + self.monitor.migrate(uri) + + if cancel_delay: + time.sleep(cancel_delay) + self.monitor.cmd("migrate_cancel") + if not virt_utils.wait_for(mig_cancelled, 60, 2, 2, + "Waiting for migration " + "cancellation"): + raise virt_vm.VMMigrateCancelError("Cannot cancel migration") + return + + wait_for_migration() + + # Report migration status + if mig_succeeded(): + logging.info("Migration completed successfully") + elif mig_failed(): + raise virt_vm.VMMigrateFailedError("Migration failed") + else: + raise virt_vm.VMMigrateFailedError("Migration ended with " + "unknown status") + + # Switch self <-> clone + temp = self.clone(copy_state=True) + self.__dict__ = clone.__dict__ + clone = temp + + # From now on, clone is the source VM that will soon be destroyed + # and self is the destination VM that will remain alive. If this + # is remote migration, self is a dead VM object. + + error.context("after migration") + if local: + time.sleep(1) + self.verify_alive() + + if local and stable_check: + try: + save1 = os.path.join(save_path, "src-" + clone.instance) + save2 = os.path.join(save_path, "dst-" + self.instance) + clone.save_to_file(save1) + self.save_to_file(save2) + # Fail if we see deltas + md5_save1 = utils.hash_file(save1) + md5_save2 = utils.hash_file(save2) + if md5_save1 != md5_save2: + raise virt_vm.VMMigrateStateMismatchError(md5_save1, + md5_save2) + finally: + if clean: + if os.path.isfile(save1): + os.remove(save1) + if os.path.isfile(save2): + os.remove(save2) + + finally: + # If we're doing remote migration and it's completed successfully, + # self points to a dead VM object + if self.is_alive(): + self.monitor.cmd("cont") + clone.destroy(gracefully=False) + + + @error.context_aware + def reboot(self, session=None, method="shell", nic_index=0, timeout=240): + """ + Reboot the VM and wait for it to come back up by trying to log in until + timeout expires. + + @param session: A shell session object or None. + @param method: Reboot method. Can be "shell" (send a shell reboot + command) or "system_reset" (send a system_reset monitor command). + @param nic_index: Index of NIC to access in the VM, when logging in + after rebooting. + @param timeout: Time to wait for login to succeed (after rebooting). + @return: A new shell session object. + """ + error.base_context("rebooting '%s'" % self.name, logging.info) + error.context("before reboot") + session = session or self.login() + error.context() + + if method == "shell": + session.sendline(self.params.get("reboot_command")) + elif method == "system_reset": + # Clear the event list of all QMP monitors + qmp_monitors = [m for m in self.monitors if m.protocol == "qmp"] + for m in qmp_monitors: + m.clear_events() + # Send a system_reset monitor command + self.monitor.cmd("system_reset") + # Look for RESET QMP events + time.sleep(1) + for m in qmp_monitors: + if m.get_event("RESET"): + logging.info("RESET QMP event received") + else: + raise virt_vm.VMRebootError("RESET QMP event not received " + "after system_reset " + "(monitor '%s')" % m.name) + else: + raise virt_vm.VMRebootError("Unknown reboot method: %s" % method) + + error.context("waiting for guest to go down", logging.info) + if not virt_utils.wait_for(lambda: + not session.is_responsive(timeout=30), + 120, 0, 1): + raise virt_vm.VMRebootError("Guest refuses to go down") + session.close() + + error.context("logging in after reboot", logging.info) + return self.wait_for_login(nic_index, timeout=timeout) + + + def send_key(self, keystr): + """ + Send a key event to the VM. + + @param: keystr: A key event string (e.g. "ctrl-alt-delete") + """ + # For compatibility with versions of QEMU that do not recognize all + # key names: replace keyname with the hex value from the dict, which + # QEMU will definitely accept + dict = {"comma": "0x33", + "dot": "0x34", + "slash": "0x35"} + for key, value in dict.items(): + keystr = keystr.replace(key, value) + self.monitor.sendkey(keystr) + time.sleep(0.2) + + + def send_string(self, str): + """ + Send a string to the VM. + + @param str: String, that must consist of alphanumeric characters only. + Capital letters are allowed. + """ + for char in str: + if char.isupper(): + self.send_key("shift-%s" % char.lower()) + else: + self.send_key(char) + + + def screendump(self, filename): + try: + if self.monitor: + self.monitor.screendump(filename=filename) + except kvm_monitor.MonitorError, e: + logging.warn(e) + + + def get_uuid(self): + """ + Catch UUID of the VM. + + @return: None,if not specified in config file + """ + if self.params.get("uuid") == "random": + return self.uuid + else: + return self.params.get("uuid", None) + + + def get_cpu_count(self): + """ + Get the cpu count of the VM. + """ + session = self.login() + try: + return int(session.cmd(self.params.get("cpu_chk_cmd"))) + finally: + session.close() + + + def get_memory_size(self, cmd=None): + """ + Get bootup memory size of the VM. + + @param check_cmd: Command used to check memory. If not provided, + self.params.get("mem_chk_cmd") will be used. + """ + session = self.login() + try: + if not cmd: + cmd = self.params.get("mem_chk_cmd") + mem_str = session.cmd(cmd) + mem = re.findall("([0-9]+)", mem_str) + mem_size = 0 + for m in mem: + mem_size += int(m) + if "GB" in mem_str: + mem_size *= 1024 + elif "MB" in mem_str: + pass + else: + mem_size /= 1024 + return int(mem_size) + finally: + session.close() + + + def get_current_memory_size(self): + """ + Get current memory size of the VM, rather than bootup memory. + """ + cmd = self.params.get("mem_chk_cur_cmd") + return self.get_memory_size(cmd) + + + def save_to_file(self, path): + """ + Save the state of virtual machine to a file through migrate to + exec + """ + # Make sure we only get one iteration + self.monitor.cmd("migrate_set_speed 1000g") + self.monitor.cmd("migrate_set_downtime 100000000") + self.monitor.migrate('"exec:cat>%s"' % path) + # Restore the speed and downtime of migration + self.monitor.cmd("migrate_set_speed %d" % (32<<20)) + self.monitor.cmd("migrate_set_downtime 0.03") diff --git a/client/virt/ppm_utils.py b/client/virt/ppm_utils.py new file mode 100644 index 0000000..90ff46d --- /dev/null +++ b/client/virt/ppm_utils.py @@ -0,0 +1,237 @@ +""" +Utility functions to deal with ppm (qemu screendump format) files. + +@copyright: Red Hat 2008-2009 +""" + +import os, struct, time, re +from autotest_lib.client.bin import utils + +# Some directory/filename utils, for consistency + +def find_id_for_screendump(md5sum, dir): + """ + Search dir for a PPM file whose name ends with md5sum. + + @param md5sum: md5 sum string + @param dir: Directory that holds the PPM files. + @return: The file's basename without any preceding path, e.g. + '20080101_120000_d41d8cd98f00b204e9800998ecf8427e.ppm'. + """ + try: + files = os.listdir(dir) + except OSError: + files = [] + for file in files: + exp = re.compile(r"(.*_)?" + md5sum + r"\.ppm", re.IGNORECASE) + if exp.match(file): + return file + + +def generate_id_for_screendump(md5sum, dir): + """ + Generate a unique filename using the given MD5 sum. + + @return: Only the file basename, without any preceding path. The + filename consists of the current date and time, the MD5 sum and a .ppm + extension, e.g. '20080101_120000_d41d8cd98f00b204e9800998ecf8427e.ppm'. + """ + filename = time.strftime("%Y%m%d_%H%M%S") + "_" + md5sum + ".ppm" + return filename + + +def get_data_dir(steps_filename): + """ + Return the data dir of the given steps filename. + """ + filename = os.path.basename(steps_filename) + return os.path.join(os.path.dirname(steps_filename), "..", "steps_data", + filename + "_data") + + +# Functions for working with PPM files + +def image_read_from_ppm_file(filename): + """ + Read a PPM image. + + @return: A 3 element tuple containing the width, height and data of the + image. + """ + fin = open(filename,"rb") + l1 = fin.readline() + l2 = fin.readline() + l3 = fin.readline() + data = fin.read() + fin.close() + + (w, h) = map(int, l2.split()) + return (w, h, data) + + +def image_write_to_ppm_file(filename, width, height, data): + """ + Write a PPM image with the given width, height and data. + + @param filename: PPM file path + @param width: PPM file width (pixels) + @param height: PPM file height (pixels) + """ + fout = open(filename,"wb") + fout.write("P6\n") + fout.write("%d %d\n" % (width, height)) + fout.write("255\n") + fout.write(data) + fout.close() + + +def image_crop(width, height, data, x1, y1, dx, dy): + """ + Crop an image. + + @param width: Original image width + @param height: Original image height + @param data: Image data + @param x1: Desired x coordinate of the cropped region + @param y1: Desired y coordinate of the cropped region + @param dx: Desired width of the cropped region + @param dy: Desired height of the cropped region + @return: A 3-tuple containing the width, height and data of the + cropped image. + """ + if x1 > width - 1: x1 = width - 1 + if y1 > height - 1: y1 = height - 1 + if dx > width - x1: dx = width - x1 + if dy > height - y1: dy = height - y1 + newdata = "" + index = (x1 + y1*width) * 3 + for i in range(dy): + newdata += data[index:(index+dx*3)] + index += width*3 + return (dx, dy, newdata) + + +def image_md5sum(width, height, data): + """ + Return the md5sum of an image. + + @param width: PPM file width + @param height: PPM file height + @data: PPM file data + """ + header = "P6\n%d %d\n255\n" % (width, height) + hash = utils.hash('md5', header) + hash.update(data) + return hash.hexdigest() + + +def get_region_md5sum(width, height, data, x1, y1, dx, dy, + cropped_image_filename=None): + """ + Return the md5sum of a cropped region. + + @param width: Original image width + @param height: Original image height + @param data: Image data + @param x1: Desired x coord of the cropped region + @param y1: Desired y coord of the cropped region + @param dx: Desired width of the cropped region + @param dy: Desired height of the cropped region + @param cropped_image_filename: if not None, write the resulting cropped + image to a file with this name + """ + (cw, ch, cdata) = image_crop(width, height, data, x1, y1, dx, dy) + # Write cropped image for debugging + if cropped_image_filename: + image_write_to_ppm_file(cropped_image_filename, cw, ch, cdata) + return image_md5sum(cw, ch, cdata) + + +def image_verify_ppm_file(filename): + """ + Verify the validity of a PPM file. + + @param filename: Path of the file being verified. + @return: True if filename is a valid PPM image file. This function + reads only the first few bytes of the file so it should be rather fast. + """ + try: + size = os.path.getsize(filename) + fin = open(filename, "rb") + assert(fin.readline().strip() == "P6") + (width, height) = map(int, fin.readline().split()) + assert(width > 0 and height > 0) + assert(fin.readline().strip() == "255") + size_read = fin.tell() + fin.close() + assert(size - size_read == width*height*3) + return True + except: + return False + + +def image_comparison(width, height, data1, data2): + """ + Generate a green-red comparison image from two given images. + + @param width: Width of both images + @param height: Height of both images + @param data1: Data of first image + @param data2: Data of second image + @return: A 3-element tuple containing the width, height and data of the + generated comparison image. + + @note: Input images must be the same size. + """ + newdata = "" + i = 0 + while i < width*height*3: + # Compute monochromatic value of current pixel in data1 + pixel1_str = data1[i:i+3] + temp = struct.unpack("BBB", pixel1_str) + value1 = int((temp[0] + temp[1] + temp[2]) / 3) + # Compute monochromatic value of current pixel in data2 + pixel2_str = data2[i:i+3] + temp = struct.unpack("BBB", pixel2_str) + value2 = int((temp[0] + temp[1] + temp[2]) / 3) + # Compute average of the two values + value = int((value1 + value2) / 2) + # Scale value to the upper half of the range [0, 255] + value = 128 + value / 2 + # Compare pixels + if pixel1_str == pixel2_str: + # Equal -- give the pixel a greenish hue + newpixel = [0, value, 0] + else: + # Not equal -- give the pixel a reddish hue + newpixel = [value, 0, 0] + newdata += struct.pack("BBB", newpixel[0], newpixel[1], newpixel[2]) + i += 3 + return (width, height, newdata) + + +def image_fuzzy_compare(width, height, data1, data2): + """ + Return the degree of equality of two given images. + + @param width: Width of both images + @param height: Height of both images + @param data1: Data of first image + @param data2: Data of second image + @return: Ratio equal_pixel_count / total_pixel_count. + + @note: Input images must be the same size. + """ + equal = 0.0 + different = 0.0 + i = 0 + while i < width*height*3: + pixel1_str = data1[i:i+3] + pixel2_str = data2[i:i+3] + # Compare pixels + if pixel1_str == pixel2_str: + equal += 1.0 + else: + different += 1.0 + i += 3 + return equal / (equal + different) diff --git a/client/virt/rss_client.py b/client/virt/rss_client.py new file mode 100755 index 0000000..4d00d17 --- /dev/null +++ b/client/virt/rss_client.py @@ -0,0 +1,519 @@ +#!/usr/bin/python +""" +Client for file transfer services offered by RSS (Remote Shell Server). + +@author: Michael Goldish (mgoldish@redhat.com) +@copyright: 2008-2010 Red Hat Inc. +""" + +import socket, struct, time, sys, os, glob + +# Globals +CHUNKSIZE = 65536 + +# Protocol message constants +RSS_MAGIC = 0x525353 +RSS_OK = 1 +RSS_ERROR = 2 +RSS_UPLOAD = 3 +RSS_DOWNLOAD = 4 +RSS_SET_PATH = 5 +RSS_CREATE_FILE = 6 +RSS_CREATE_DIR = 7 +RSS_LEAVE_DIR = 8 +RSS_DONE = 9 + +# See rss.cpp for protocol details. + + +class FileTransferError(Exception): + def __init__(self, msg, e=None, filename=None): + Exception.__init__(self, msg, e, filename) + self.msg = msg + self.e = e + self.filename = filename + + def __str__(self): + s = self.msg + if self.e and self.filename: + s += " (error: %s, filename: %s)" % (self.e, self.filename) + elif self.e: + s += " (%s)" % self.e + elif self.filename: + s += " (filename: %s)" % self.filename + return s + + +class FileTransferConnectError(FileTransferError): + pass + + +class FileTransferTimeoutError(FileTransferError): + pass + + +class FileTransferProtocolError(FileTransferError): + pass + + +class FileTransferSocketError(FileTransferError): + pass + + +class FileTransferServerError(FileTransferError): + def __init__(self, errmsg): + FileTransferError.__init__(self, None, errmsg) + + def __str__(self): + s = "Server said: %r" % self.e + if self.filename: + s += " (filename: %s)" % self.filename + return s + + +class FileTransferNotFoundError(FileTransferError): + pass + + +class FileTransferClient(object): + """ + Connect to a RSS (remote shell server) and transfer files. + """ + + def __init__(self, address, port, log_func=None, timeout=20): + """ + Connect to a server. + + @param address: The server's address + @param port: The server's port + @param log_func: If provided, transfer stats will be passed to this + function during the transfer + @param timeout: Time duration to wait for connection to succeed + @raise FileTransferConnectError: Raised if the connection fails + """ + self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._socket.settimeout(timeout) + try: + self._socket.connect((address, port)) + except socket.error, e: + raise FileTransferConnectError("Cannot connect to server at " + "%s:%s" % (address, port), e) + try: + if self._receive_msg(timeout) != RSS_MAGIC: + raise FileTransferConnectError("Received wrong magic number") + except FileTransferTimeoutError: + raise FileTransferConnectError("Timeout expired while waiting to " + "receive magic number") + self._send(struct.pack("=i", CHUNKSIZE)) + self._log_func = log_func + self._last_time = time.time() + self._last_transferred = 0 + self.transferred = 0 + + + def __del__(self): + self.close() + + + def close(self): + """ + Close the connection. + """ + self._socket.close() + + + def _send(self, str, timeout=60): + try: + if timeout <= 0: + raise socket.timeout + self._socket.settimeout(timeout) + self._socket.sendall(str) + except socket.timeout: + raise FileTransferTimeoutError("Timeout expired while sending " + "data to server") + except socket.error, e: + raise FileTransferSocketError("Could not send data to server", e) + + + def _receive(self, size, timeout=60): + strs = [] + end_time = time.time() + timeout + try: + while size > 0: + timeout = end_time - time.time() + if timeout <= 0: + raise socket.timeout + self._socket.settimeout(timeout) + data = self._socket.recv(size) + if not data: + raise FileTransferProtocolError("Connection closed " + "unexpectedly while " + "receiving data from " + "server") + strs.append(data) + size -= len(data) + except socket.timeout: + raise FileTransferTimeoutError("Timeout expired while receiving " + "data from server") + except socket.error, e: + raise FileTransferSocketError("Error receiving data from server", + e) + return "".join(strs) + + + def _report_stats(self, str): + if self._log_func: + dt = time.time() - self._last_time + if dt >= 1: + transferred = self.transferred / 1048576. + speed = (self.transferred - self._last_transferred) / dt + speed /= 1048576. + self._log_func("%s %.3f MB (%.3f MB/sec)" % + (str, transferred, speed)) + self._last_time = time.time() + self._last_transferred = self.transferred + + + def _send_packet(self, str, timeout=60): + self._send(struct.pack("=I", len(str))) + self._send(str, timeout) + self.transferred += len(str) + 4 + self._report_stats("Sent") + + + def _receive_packet(self, timeout=60): + size = struct.unpack("=I", self._receive(4))[0] + str = self._receive(size, timeout) + self.transferred += len(str) + 4 + self._report_stats("Received") + return str + + + def _send_file_chunks(self, filename, timeout=60): + if self._log_func: + self._log_func("Sending file %s" % filename) + f = open(filename, "rb") + try: + try: + end_time = time.time() + timeout + while True: + data = f.read(CHUNKSIZE) + self._send_packet(data, end_time - time.time()) + if len(data) < CHUNKSIZE: + break + except FileTransferError, e: + e.filename = filename + raise + finally: + f.close() + + + def _receive_file_chunks(self, filename, timeout=60): + if self._log_func: + self._log_func("Receiving file %s" % filename) + f = open(filename, "wb") + try: + try: + end_time = time.time() + timeout + while True: + data = self._receive_packet(end_time - time.time()) + f.write(data) + if len(data) < CHUNKSIZE: + break + except FileTransferError, e: + e.filename = filename + raise + finally: + f.close() + + + def _send_msg(self, msg, timeout=60): + self._send(struct.pack("=I", msg)) + + + def _receive_msg(self, timeout=60): + s = self._receive(4, timeout) + return struct.unpack("=I", s)[0] + + + def _handle_transfer_error(self): + # Save original exception + e = sys.exc_info() + try: + # See if we can get an error message + msg = self._receive_msg() + except FileTransferError: + # No error message -- re-raise original exception + raise e[0], e[1], e[2] + if msg == RSS_ERROR: + errmsg = self._receive_packet() + raise FileTransferServerError(errmsg) + raise e[0], e[1], e[2] + + +class FileUploadClient(FileTransferClient): + """ + Connect to a RSS (remote shell server) and upload files or directory trees. + """ + + def __init__(self, address, port, log_func=None, timeout=20): + """ + Connect to a server. + + @param address: The server's address + @param port: The server's port + @param log_func: If provided, transfer stats will be passed to this + function during the transfer + @param timeout: Time duration to wait for connection to succeed + @raise FileTransferConnectError: Raised if the connection fails + @raise FileTransferProtocolError: Raised if an incorrect magic number + is received + @raise FileTransferSocketError: Raised if the RSS_UPLOAD message cannot + be sent to the server + """ + super(FileUploadClient, self).__init__(address, port, log_func, timeout) + self._send_msg(RSS_UPLOAD) + + + def _upload_file(self, path, end_time): + if os.path.isfile(path): + self._send_msg(RSS_CREATE_FILE) + self._send_packet(os.path.basename(path)) + self._send_file_chunks(path, end_time - time.time()) + elif os.path.isdir(path): + self._send_msg(RSS_CREATE_DIR) + self._send_packet(os.path.basename(path)) + for filename in os.listdir(path): + self._upload_file(os.path.join(path, filename), end_time) + self._send_msg(RSS_LEAVE_DIR) + + + def upload(self, src_pattern, dst_path, timeout=600): + """ + Send files or directory trees to the server. + The semantics of src_pattern and dst_path are similar to those of scp. + For example, the following are OK: + src_pattern='/tmp/foo.txt', dst_path='C:\\' + (uploads a single file) + src_pattern='/usr/', dst_path='C:\\Windows\\' + (uploads a directory tree recursively) + src_pattern='/usr/*', dst_path='C:\\Windows\\' + (uploads all files and directory trees under /usr/) + The following is not OK: + src_pattern='/tmp/foo.txt', dst_path='C:\\Windows\\*' + (wildcards are only allowed in src_pattern) + + @param src_pattern: A path or wildcard pattern specifying the files or + directories to send to the server + @param dst_path: A path in the server's filesystem where the files will + be saved + @param timeout: Time duration in seconds to wait for the transfer to + complete + @raise FileTransferTimeoutError: Raised if timeout expires + @raise FileTransferServerError: Raised if something goes wrong and the + server sends an informative error message to the client + @note: Other exceptions can be raised. + """ + end_time = time.time() + timeout + try: + try: + self._send_msg(RSS_SET_PATH) + self._send_packet(dst_path) + matches = glob.glob(src_pattern) + for filename in matches: + self._upload_file(os.path.abspath(filename), end_time) + self._send_msg(RSS_DONE) + except FileTransferTimeoutError: + raise + except FileTransferError: + self._handle_transfer_error() + else: + # If nothing was transferred, raise an exception + if not matches: + raise FileTransferNotFoundError("Pattern %s does not " + "match any files or " + "directories" % + src_pattern) + # Look for RSS_OK or RSS_ERROR + msg = self._receive_msg(end_time - time.time()) + if msg == RSS_OK: + return + elif msg == RSS_ERROR: + errmsg = self._receive_packet() + raise FileTransferServerError(errmsg) + else: + # Neither RSS_OK nor RSS_ERROR found + raise FileTransferProtocolError("Received unexpected msg") + except: + # In any case, if the transfer failed, close the connection + self.close() + raise + + +class FileDownloadClient(FileTransferClient): + """ + Connect to a RSS (remote shell server) and download files or directory trees. + """ + + def __init__(self, address, port, log_func=None, timeout=20): + """ + Connect to a server. + + @param address: The server's address + @param port: The server's port + @param log_func: If provided, transfer stats will be passed to this + function during the transfer + @param timeout: Time duration to wait for connection to succeed + @raise FileTransferConnectError: Raised if the connection fails + @raise FileTransferProtocolError: Raised if an incorrect magic number + is received + @raise FileTransferSendError: Raised if the RSS_UPLOAD message cannot + be sent to the server + """ + super(FileDownloadClient, self).__init__(address, port, log_func, timeout) + self._send_msg(RSS_DOWNLOAD) + + + def download(self, src_pattern, dst_path, timeout=600): + """ + Receive files or directory trees from the server. + The semantics of src_pattern and dst_path are similar to those of scp. + For example, the following are OK: + src_pattern='C:\\foo.txt', dst_path='/tmp' + (downloads a single file) + src_pattern='C:\\Windows', dst_path='/tmp' + (downloads a directory tree recursively) + src_pattern='C:\\Windows\\*', dst_path='/tmp' + (downloads all files and directory trees under C:\\Windows) + The following is not OK: + src_pattern='C:\\Windows', dst_path='/tmp/*' + (wildcards are only allowed in src_pattern) + + @param src_pattern: A path or wildcard pattern specifying the files or + directories, in the server's filesystem, that will be sent to + the client + @param dst_path: A path in the local filesystem where the files will + be saved + @param timeout: Time duration in seconds to wait for the transfer to + complete + @raise FileTransferTimeoutError: Raised if timeout expires + @raise FileTransferServerError: Raised if something goes wrong and the + server sends an informative error message to the client + @note: Other exceptions can be raised. + """ + dst_path = os.path.abspath(dst_path) + end_time = time.time() + timeout + file_count = 0 + dir_count = 0 + try: + try: + self._send_msg(RSS_SET_PATH) + self._send_packet(src_pattern) + except FileTransferError: + self._handle_transfer_error() + while True: + msg = self._receive_msg() + if msg == RSS_CREATE_FILE: + # Receive filename and file contents + filename = self._receive_packet() + if os.path.isdir(dst_path): + dst_path = os.path.join(dst_path, filename) + self._receive_file_chunks(dst_path, end_time - time.time()) + dst_path = os.path.dirname(dst_path) + file_count += 1 + elif msg == RSS_CREATE_DIR: + # Receive dirname and create the directory + dirname = self._receive_packet() + if os.path.isdir(dst_path): + dst_path = os.path.join(dst_path, dirname) + if not os.path.isdir(dst_path): + os.mkdir(dst_path) + dir_count += 1 + elif msg == RSS_LEAVE_DIR: + # Return to parent dir + dst_path = os.path.dirname(dst_path) + elif msg == RSS_DONE: + # Transfer complete + if not file_count and not dir_count: + raise FileTransferNotFoundError("Pattern %s does not " + "match any files or " + "directories that " + "could be downloaded" % + src_pattern) + break + elif msg == RSS_ERROR: + # Receive error message and abort + errmsg = self._receive_packet() + raise FileTransferServerError(errmsg) + else: + # Unexpected msg + raise FileTransferProtocolError("Received unexpected msg") + except: + # In any case, if the transfer failed, close the connection + self.close() + raise + + +def upload(address, port, src_pattern, dst_path, log_func=None, timeout=60, + connect_timeout=20): + """ + Connect to server and upload files. + + @see: FileUploadClient + """ + client = FileUploadClient(address, port, log_func, connect_timeout) + client.upload(src_pattern, dst_path, timeout) + client.close() + + +def download(address, port, src_pattern, dst_path, log_func=None, timeout=60, + connect_timeout=20): + """ + Connect to server and upload files. + + @see: FileDownloadClient + """ + client = FileDownloadClient(address, port, log_func, connect_timeout) + client.download(src_pattern, dst_path, timeout) + client.close() + + +def main(): + import optparse + + usage = "usage: %prog [options] address port src_pattern dst_path" + parser = optparse.OptionParser(usage=usage) + parser.add_option("-d", "--download", + action="store_true", dest="download", + help="download files from server") + parser.add_option("-u", "--upload", + action="store_true", dest="upload", + help="upload files to server") + parser.add_option("-v", "--verbose", + action="store_true", dest="verbose", + help="be verbose") + parser.add_option("-t", "--timeout", + type="int", dest="timeout", default=3600, + help="transfer timeout") + options, args = parser.parse_args() + if options.download == options.upload: + parser.error("you must specify either -d or -u") + if len(args) != 4: + parser.error("incorrect number of arguments") + address, port, src_pattern, dst_path = args + port = int(port) + + logger = None + if options.verbose: + def p(s): + print s + logger = p + + if options.download: + download(address, port, src_pattern, dst_path, logger, options.timeout) + elif options.upload: + upload(address, port, src_pattern, dst_path, logger, options.timeout) + + +if __name__ == "__main__": + main() diff --git a/client/virt/virt_env_process.py b/client/virt/virt_env_process.py new file mode 100644 index 0000000..ca00528 --- /dev/null +++ b/client/virt/virt_env_process.py @@ -0,0 +1,438 @@ +import os, time, commands, re, logging, glob, threading, shutil +from autotest_lib.client.bin import utils +from autotest_lib.client.common_lib import error +import virt_utils, virt_vm, kvm_monitor, virt_test_setup, ppm_utils, aexpect +try: + import PIL.Image +except ImportError: + logging.warning('No python imaging library installed. PPM image ' + 'conversion to JPEG disabled. In order to enable it, ' + 'please install python-imaging or the equivalent for your ' + 'distro.') + + +_screendump_thread = None +_screendump_thread_termination_event = None + + +def preprocess_image(test, params): + """ + Preprocess a single QEMU image according to the instructions in params. + + @param test: Autotest test object. + @param params: A dict containing image preprocessing parameters. + @note: Currently this function just creates an image if requested. + """ + image_filename = virt_vm.get_image_filename(params, test.bindir) + + create_image = False + + if params.get("force_create_image") == "yes": + logging.debug("'force_create_image' specified; creating image...") + create_image = True + elif (params.get("create_image") == "yes" and not + os.path.exists(image_filename)): + logging.debug("Creating image...") + create_image = True + + if create_image and not virt_vm.create_image(params, test.bindir): + raise error.TestError("Could not create image") + + +def preprocess_vm(test, params, env, name): + """ + Preprocess a single VM object according to the instructions in params. + Start the VM if requested and get a screendump. + + @param test: An Autotest test object. + @param params: A dict containing VM preprocessing parameters. + @param env: The environment (a dict-like object). + @param name: The name of the VM object. + """ + logging.debug("Preprocessing VM '%s'..." % name) + vm = env.get_vm(name) + vm_type = params.get("vm_type") + if not vm: + logging.debug("VM object does not exist; creating it") + vm = virt_vm.instantiate_vm(vm_type=vm_type, name=name, params=params, + root_dir=test.bindir, + address_cache=env.get("address_cache")) + env.register_vm(name, vm) + + start_vm = False + + if params.get("restart_vm") == "yes": + logging.debug("'restart_vm' specified; (re)starting VM...") + start_vm = True + elif params.get("migration_mode"): + logging.debug("Starting VM in incoming migration mode...") + start_vm = True + elif params.get("start_vm") == "yes": + if not vm.is_alive(): + logging.debug("VM is not alive; starting it...") + start_vm = True + elif vm.make_qemu_command() != vm.make_qemu_command(name, params, + test.bindir): + logging.debug("VM's qemu command differs from requested one; " + "restarting it...") + start_vm = True + + if start_vm: + # Start the VM (or restart it if it's already up) + vm.create(name, params, test.bindir, + migration_mode=params.get("migration_mode")) + else: + # Don't start the VM, just update its params + vm.params = params + + scrdump_filename = os.path.join(test.debugdir, "pre_%s.ppm" % name) + try: + if vm.monitor: + vm.monitor.screendump(scrdump_filename) + except kvm_monitor.MonitorError, e: + logging.warn(e) + + +def postprocess_image(test, params): + """ + Postprocess a single image according to the instructions in params. + + @param test: An Autotest test object. + @param params: A dict containing image postprocessing parameters. + """ + if params.get("check_image") == "yes": + virt_vm.check_image(params, test.bindir) + if params.get("remove_image") == "yes": + virt_vm.remove_image(params, test.bindir) + + +def postprocess_vm(test, params, env, name): + """ + Postprocess a single VM object according to the instructions in params. + Kill the VM if requested and get a screendump. + + @param test: An Autotest test object. + @param params: A dict containing VM postprocessing parameters. + @param env: The environment (a dict-like object). + @param name: The name of the VM object. + """ + logging.debug("Postprocessing VM '%s'..." % name) + vm = env.get_vm(name) + if not vm: + return + + scrdump_filename = os.path.join(test.debugdir, "post_%s.ppm" % name) + vm.screendump(filename=scrdump_filename) + + if params.get("kill_vm") == "yes": + kill_vm_timeout = float(params.get("kill_vm_timeout", 0)) + if kill_vm_timeout: + logging.debug("'kill_vm' specified; waiting for VM to shut down " + "before killing it...") + virt_utils.wait_for(vm.is_dead, kill_vm_timeout, 0, 1) + else: + logging.debug("'kill_vm' specified; killing VM...") + vm.destroy(gracefully = params.get("kill_vm_gracefully") == "yes") + + +def process_command(test, params, env, command, command_timeout, + command_noncritical): + """ + Pre- or post- custom commands to be executed before/after a test is run + + @param test: An Autotest test object. + @param params: A dict containing all VM and image parameters. + @param env: The environment (a dict-like object). + @param command: Command to be run. + @param command_timeout: Timeout for command execution. + @param command_noncritical: If True test will not fail if command fails. + """ + # Export environment vars + for k in params: + os.putenv("VIRT_TEST_%s" % k, str(params[k])) + # Execute commands + try: + utils.system("cd %s; %s" % (test.bindir, command)) + except error.CmdError, e: + if command_noncritical: + logging.warn(e) + else: + raise + +def process(test, params, env, image_func, vm_func): + """ + Pre- or post-process VMs and images according to the instructions in params. + Call image_func for each image listed in params and vm_func for each VM. + + @param test: An Autotest test object. + @param params: A dict containing all VM and image parameters. + @param env: The environment (a dict-like object). + @param image_func: A function to call for each image. + @param vm_func: A function to call for each VM. + """ + # Get list of VMs specified for this test + for vm_name in params.objects("vms"): + vm_params = params.object_params(vm_name) + # Get list of images specified for this VM + for image_name in vm_params.objects("images"): + image_params = vm_params.object_params(image_name) + # Call image_func for each image + image_func(test, image_params) + # Call vm_func for each vm + vm_func(test, vm_params, env, vm_name) + + +@error.context_aware +def preprocess(test, params, env): + """ + Preprocess all VMs and images according to the instructions in params. + Also, collect some host information, such as the KVM version. + + @param test: An Autotest test object. + @param params: A dict containing all VM and image parameters. + @param env: The environment (a dict-like object). + """ + error.context("preprocessing") + + # Start tcpdump if it isn't already running + if "address_cache" not in env: + env["address_cache"] = {} + if "tcpdump" in env and not env["tcpdump"].is_alive(): + env["tcpdump"].close() + del env["tcpdump"] + if "tcpdump" not in env and params.get("run_tcpdump", "yes") == "yes": + cmd = "%s -npvi any 'dst port 68'" % virt_utils.find_command("tcpdump") + logging.debug("Starting tcpdump (%s)...", cmd) + env["tcpdump"] = aexpect.Tail( + command=cmd, + output_func=_update_address_cache, + output_params=(env["address_cache"],)) + if virt_utils.wait_for(lambda: not env["tcpdump"].is_alive(), + 0.1, 0.1, 1.0): + logging.warn("Could not start tcpdump") + logging.warn("Status: %s" % env["tcpdump"].get_status()) + logging.warn("Output:" + virt_utils.format_str_for_message( + env["tcpdump"].get_output())) + + # Destroy and remove VMs that are no longer needed in the environment + requested_vms = params.objects("vms") + for key in env.keys(): + vm = env[key] + if not virt_utils.is_vm(vm): + continue + if not vm.name in requested_vms: + logging.debug("VM '%s' found in environment but not required for " + "test; removing it..." % vm.name) + vm.destroy() + del env[key] + + virt_utils.get_virt_info(params, test) + + if params.get("setup_hugepages") == "yes": + h = virt_test_setup.HugePageConfig(params) + h.setup() + + if params.get("type") == "unattended_install": + u = virt_test_setup.UnattendedInstallConfig(test, params) + u.setup() + + if params.get("type") == "enospc": + e = virt_test_setup.EnospcConfig(test, params) + e.setup() + + # Execute any pre_commands + if params.get("pre_command"): + process_command(test, params, env, params.get("pre_command"), + int(params.get("pre_command_timeout", "600")), + params.get("pre_command_noncritical") == "yes") + + # Preprocess all VMs and images + process(test, params, env, preprocess_image, preprocess_vm) + + # Start the screendump thread + if params.get("take_regular_screendumps") == "yes": + logging.debug("Starting screendump thread") + global _screendump_thread, _screendump_thread_termination_event + _screendump_thread_termination_event = threading.Event() + _screendump_thread = threading.Thread(target=_take_screendumps, + args=(test, params, env)) + _screendump_thread.start() + + +@error.context_aware +def postprocess(test, params, env): + """ + Postprocess all VMs and images according to the instructions in params. + + @param test: An Autotest test object. + @param params: Dict containing all VM and image parameters. + @param env: The environment (a dict-like object). + """ + error.context("postprocessing") + + # Postprocess all VMs and images + process(test, params, env, postprocess_image, postprocess_vm) + + # Terminate the screendump thread + global _screendump_thread, _screendump_thread_termination_event + if _screendump_thread: + logging.debug("Terminating screendump thread...") + _screendump_thread_termination_event.set() + _screendump_thread.join(10) + _screendump_thread = None + + # Warn about corrupt PPM files + for f in glob.glob(os.path.join(test.debugdir, "*.ppm")): + if not ppm_utils.image_verify_ppm_file(f): + logging.warn("Found corrupt PPM file: %s", f) + + # Should we convert PPM files to PNG format? + if params.get("convert_ppm_files_to_png") == "yes": + logging.debug("'convert_ppm_files_to_png' specified; converting PPM " + "files to PNG format...") + try: + for f in glob.glob(os.path.join(test.debugdir, "*.ppm")): + if ppm_utils.image_verify_ppm_file(f): + new_path = f.replace(".ppm", ".png") + image = PIL.Image.open(f) + image.save(new_path, format='PNG') + except NameError: + pass + + # Should we keep the PPM files? + if params.get("keep_ppm_files") != "yes": + logging.debug("'keep_ppm_files' not specified; removing all PPM files " + "from debug dir...") + for f in glob.glob(os.path.join(test.debugdir, '*.ppm')): + os.unlink(f) + + # Should we keep the screendump dirs? + if params.get("keep_screendumps") != "yes": + logging.debug("'keep_screendumps' not specified; removing screendump " + "dirs...") + for d in glob.glob(os.path.join(test.debugdir, "screendumps_*")): + if os.path.isdir(d) and not os.path.islink(d): + shutil.rmtree(d, ignore_errors=True) + + # Kill all unresponsive VMs + if params.get("kill_unresponsive_vms") == "yes": + logging.debug("'kill_unresponsive_vms' specified; killing all VMs " + "that fail to respond to a remote login request...") + for vm in env.get_all_vms(): + if vm.is_alive(): + try: + session = vm.login() + session.close() + except (virt_utils.LoginError, virt_vm.VMError), e: + logging.warn(e) + vm.destroy(gracefully=False) + + # Kill all aexpect tail threads + aexpect.kill_tail_threads() + + # Terminate tcpdump if no VMs are alive + living_vms = [vm for vm in env.get_all_vms() if vm.is_alive()] + if not living_vms and "tcpdump" in env: + env["tcpdump"].close() + del env["tcpdump"] + + if params.get("setup_hugepages") == "yes": + h = virt_test_setup.HugePageConfig(params) + h.cleanup() + + if params.get("type") == "enospc": + e = virt_test_setup.EnospcConfig(test, params) + e.cleanup() + + # Execute any post_commands + if params.get("post_command"): + process_command(test, params, env, params.get("post_command"), + int(params.get("post_command_timeout", "600")), + params.get("post_command_noncritical") == "yes") + + +def postprocess_on_error(test, params, env): + """ + Perform postprocessing operations required only if the test failed. + + @param test: An Autotest test object. + @param params: A dict containing all VM and image parameters. + @param env: The environment (a dict-like object). + """ + params.update(params.object_params("on_error")) + + +def _update_address_cache(address_cache, line): + if re.search("Your.IP", line, re.IGNORECASE): + matches = re.findall(r"\d*\.\d*\.\d*\.\d*", line) + if matches: + address_cache["last_seen"] = matches[0] + if re.search("Client.Ethernet.Address", line, re.IGNORECASE): + matches = re.findall(r"\w*:\w*:\w*:\w*:\w*:\w*", line) + if matches and address_cache.get("last_seen"): + mac_address = matches[0].lower() + if time.time() - address_cache.get("time_%s" % mac_address, 0) > 5: + logging.debug("(address cache) Adding cache entry: %s ---> %s", + mac_address, address_cache.get("last_seen")) + address_cache[mac_address] = address_cache.get("last_seen") + address_cache["time_%s" % mac_address] = time.time() + del address_cache["last_seen"] + + +def _take_screendumps(test, params, env): + global _screendump_thread_termination_event + temp_dir = test.debugdir + if params.get("screendump_temp_dir"): + temp_dir = virt_utils.get_path(test.bindir, + params.get("screendump_temp_dir")) + try: + os.makedirs(temp_dir) + except OSError: + pass + temp_filename = os.path.join(temp_dir, "scrdump-%s.ppm" % + virt_utils.generate_random_string(6)) + delay = float(params.get("screendump_delay", 5)) + quality = int(params.get("screendump_quality", 30)) + + cache = {} + + while True: + for vm in env.get_all_vms(): + if not vm.is_alive(): + continue + vm.screendump(filename=temp_filename) + if not os.path.exists(temp_filename): + logging.warn("VM '%s' failed to produce a screendump", vm.name) + continue + if not ppm_utils.image_verify_ppm_file(temp_filename): + logging.warn("VM '%s' produced an invalid screendump", vm.name) + os.unlink(temp_filename) + continue + screendump_dir = os.path.join(test.debugdir, + "screendumps_%s" % vm.name) + try: + os.makedirs(screendump_dir) + except OSError: + pass + screendump_filename = os.path.join(screendump_dir, + "%s_%s.jpg" % (vm.name, + time.strftime("%Y-%m-%d_%H-%M-%S"))) + hash = utils.hash_file(temp_filename) + if hash in cache: + try: + os.link(cache[hash], screendump_filename) + except OSError: + pass + else: + try: + image = PIL.Image.open(temp_filename) + image.save(screendump_filename, format="JPEG", + quality=quality) + cache[hash] = screendump_filename + except NameError: + pass + os.unlink(temp_filename) + if _screendump_thread_termination_event.isSet(): + _screendump_thread_termination_event = None + break + _screendump_thread_termination_event.wait(delay) diff --git a/client/virt/virt_scheduler.py b/client/virt/virt_scheduler.py new file mode 100644 index 0000000..09427f9 --- /dev/null +++ b/client/virt/virt_scheduler.py @@ -0,0 +1,229 @@ +import os, select +import virt_utils, kvm_vm, aexpect + + +class scheduler: + """ + A scheduler that manages several parallel test execution pipelines on a + single host. + """ + + def __init__(self, tests, num_workers, total_cpus, total_mem, bindir): + """ + Initialize the class. + + @param tests: A list of test dictionaries. + @param num_workers: The number of workers (pipelines). + @param total_cpus: The total number of CPUs to dedicate to tests. + @param total_mem: The total amount of memory to dedicate to tests. + @param bindir: The directory where environment files reside. + """ + self.tests = tests + self.num_workers = num_workers + self.total_cpus = total_cpus + self.total_mem = total_mem + self.bindir = bindir + # Pipes -- s stands for scheduler, w stands for worker + self.s2w = [os.pipe() for i in range(num_workers)] + self.w2s = [os.pipe() for i in range(num_workers)] + self.s2w_r = [os.fdopen(r, "r", 0) for r, w in self.s2w] + self.s2w_w = [os.fdopen(w, "w", 0) for r, w in self.s2w] + self.w2s_r = [os.fdopen(r, "r", 0) for r, w in self.w2s] + self.w2s_w = [os.fdopen(w, "w", 0) for r, w in self.w2s] + # "Personal" worker dicts contain modifications that are applied + # specifically to each worker. For example, each worker must use a + # different environment file and a different MAC address pool. + self.worker_dicts = [{"env": "env%d" % i} for i in range(num_workers)] + + + def worker(self, index, run_test_func): + """ + The worker function. + + Waits for commands from the scheduler and processes them. + + @param index: The index of this worker (in the range 0..num_workers-1). + @param run_test_func: A function to be called to run a test + (e.g. job.run_test). + """ + r = self.s2w_r[index] + w = self.w2s_w[index] + self_dict = self.worker_dicts[index] + + # Inform the scheduler this worker is ready + w.write("ready\n") + + while True: + cmd = r.readline().split() + if not cmd: + continue + + # The scheduler wants this worker to run a test + if cmd[0] == "run": + test_index = int(cmd[1]) + test = self.tests[test_index].copy() + test.update(self_dict) + test_iterations = int(test.get("iterations", 1)) + status = run_test_func("kvm", params=test, + tag=test.get("shortname"), + iterations=test_iterations) + w.write("done %s %s\n" % (test_index, status)) + w.write("ready\n") + + # The scheduler wants this worker to free its used resources + elif cmd[0] == "cleanup": + env_filename = os.path.join(self.bindir, self_dict["env"]) + env = virt_utils.Env(env_filename) + for obj in env.values(): + if isinstance(obj, kvm_vm.VM): + obj.destroy() + elif isinstance(obj, aexpect.Spawn): + obj.close() + env.save() + w.write("cleanup_done\n") + w.write("ready\n") + + # There's no more work for this worker + elif cmd[0] == "terminate": + break + + + def scheduler(self): + """ + The scheduler function. + + Sends commands to workers, telling them to run tests, clean up or + terminate execution. + """ + idle_workers = [] + closing_workers = [] + test_status = ["waiting"] * len(self.tests) + test_worker = [None] * len(self.tests) + used_cpus = [0] * self.num_workers + used_mem = [0] * self.num_workers + + while True: + # Wait for a message from a worker + r, w, x = select.select(self.w2s_r, [], []) + + someone_is_ready = False + + for pipe in r: + worker_index = self.w2s_r.index(pipe) + msg = pipe.readline().split() + if not msg: + continue + + # A worker is ready -- add it to the idle_workers list + if msg[0] == "ready": + idle_workers.append(worker_index) + someone_is_ready = True + + # A worker completed a test + elif msg[0] == "done": + test_index = int(msg[1]) + test = self.tests[test_index] + status = int(eval(msg[2])) + test_status[test_index] = ("fail", "pass")[status] + # If the test failed, mark all dependent tests as "failed" too + if not status: + for i, other_test in enumerate(self.tests): + for dep in other_test.get("dep", []): + if dep in test["name"]: + test_status[i] = "fail" + + # A worker is done shutting down its VMs and other processes + elif msg[0] == "cleanup_done": + used_cpus[worker_index] = 0 + used_mem[worker_index] = 0 + closing_workers.remove(worker_index) + + if not someone_is_ready: + continue + + for worker in idle_workers[:]: + # Find a test for this worker + test_found = False + for i, test in enumerate(self.tests): + # We only want "waiting" tests + if test_status[i] != "waiting": + continue + # Make sure the test isn't assigned to another worker + if test_worker[i] is not None and test_worker[i] != worker: + continue + # Make sure the test's dependencies are satisfied + dependencies_satisfied = True + for dep in test["dep"]: + dependencies = [j for j, t in enumerate(self.tests) + if dep in t["name"]] + bad_status_deps = [j for j in dependencies + if test_status[j] != "pass"] + if bad_status_deps: + dependencies_satisfied = False + break + if not dependencies_satisfied: + continue + # Make sure we have enough resources to run the test + test_used_cpus = int(test.get("used_cpus", 1)) + test_used_mem = int(test.get("used_mem", 128)) + # First make sure the other workers aren't using too many + # CPUs (not including the workers currently shutting down) + uc = (sum(used_cpus) - used_cpus[worker] - + sum(used_cpus[i] for i in closing_workers)) + if uc and uc + test_used_cpus > self.total_cpus: + continue + # ... or too much memory + um = (sum(used_mem) - used_mem[worker] - + sum(used_mem[i] for i in closing_workers)) + if um and um + test_used_mem > self.total_mem: + continue + # If we reached this point it means there are, or will + # soon be, enough resources to run the test + test_found = True + # Now check if the test can be run right now, i.e. if the + # other workers, including the ones currently shutting + # down, aren't using too many CPUs + uc = (sum(used_cpus) - used_cpus[worker]) + if uc and uc + test_used_cpus > self.total_cpus: + continue + # ... or too much memory + um = (sum(used_mem) - used_mem[worker]) + if um and um + test_used_mem > self.total_mem: + continue + # Everything is OK -- run the test + test_status[i] = "running" + test_worker[i] = worker + idle_workers.remove(worker) + # Update used_cpus and used_mem + used_cpus[worker] = test_used_cpus + used_mem[worker] = test_used_mem + # Assign all related tests to this worker + for j, other_test in enumerate(self.tests): + for other_dep in other_test["dep"]: + # All tests that depend on this test + if other_dep in test["name"]: + test_worker[j] = worker + break + # ... and all tests that share a dependency + # with this test + for dep in test["dep"]: + if dep in other_dep or other_dep in dep: + test_worker[j] = worker + break + # Tell the worker to run the test + self.s2w_w[worker].write("run %s\n" % i) + break + + # If there won't be any tests for this worker to run soon, tell + # the worker to free its used resources + if not test_found and (used_cpus[worker] or used_mem[worker]): + self.s2w_w[worker].write("cleanup\n") + idle_workers.remove(worker) + closing_workers.append(worker) + + # If there are no more new tests to run, terminate the workers and + # the scheduler + if len(idle_workers) == self.num_workers: + for worker in idle_workers: + self.s2w_w[worker].write("terminate\n") + break diff --git a/client/virt/virt_step_editor.py b/client/virt/virt_step_editor.py new file mode 100755 index 0000000..bcdf572 --- /dev/null +++ b/client/virt/virt_step_editor.py @@ -0,0 +1,1401 @@ +#!/usr/bin/python +""" +Step file creator/editor. + +@copyright: Red Hat Inc 2009 +@author: mgoldish@redhat.com (Michael Goldish) +@version: "20090401" +""" + +import pygtk, gtk, os, glob, shutil, sys, logging +import common, ppm_utils +pygtk.require('2.0') + + +# General utilities + +def corner_and_size_clipped(startpoint, endpoint, limits): + c0 = startpoint[:] + c1 = endpoint[:] + if c0[0] < 0: + c0[0] = 0 + if c0[1] < 0: + c0[1] = 0 + if c1[0] < 0: + c1[0] = 0 + if c1[1] < 0: + c1[1] = 0 + if c0[0] > limits[0] - 1: + c0[0] = limits[0] - 1 + if c0[1] > limits[1] - 1: + c0[1] = limits[1] - 1 + if c1[0] > limits[0] - 1: + c1[0] = limits[0] - 1 + if c1[1] > limits[1] - 1: + c1[1] = limits[1] - 1 + return ([min(c0[0], c1[0]), + min(c0[1], c1[1])], + [abs(c1[0] - c0[0]) + 1, + abs(c1[1] - c0[1]) + 1]) + + +def key_event_to_qemu_string(event): + keymap = gtk.gdk.keymap_get_default() + keyvals = keymap.get_entries_for_keycode(event.hardware_keycode) + keyval = keyvals[0][0] + keyname = gtk.gdk.keyval_name(keyval) + + dict = { "Return": "ret", + "Tab": "tab", + "space": "spc", + "Left": "left", + "Right": "right", + "Up": "up", + "Down": "down", + "F1": "f1", + "F2": "f2", + "F3": "f3", + "F4": "f4", + "F5": "f5", + "F6": "f6", + "F7": "f7", + "F8": "f8", + "F9": "f9", + "F10": "f10", + "F11": "f11", + "F12": "f12", + "Escape": "esc", + "minus": "minus", + "equal": "equal", + "BackSpace": "backspace", + "comma": "comma", + "period": "dot", + "slash": "slash", + "Insert": "insert", + "Delete": "delete", + "Home": "home", + "End": "end", + "Page_Up": "pgup", + "Page_Down": "pgdn", + "Menu": "menu", + "semicolon": "0x27", + "backslash": "0x2b", + "apostrophe": "0x28", + "grave": "0x29", + "less": "0x2b", + "bracketleft": "0x1a", + "bracketright": "0x1b", + "Super_L": "0xdc", + "Super_R": "0xdb", + } + + if ord('a') <= keyval <= ord('z') or ord('0') <= keyval <= ord('9'): + str = keyname + elif keyname in dict.keys(): + str = dict[keyname] + else: + return "" + + if event.state & gtk.gdk.CONTROL_MASK: + str = "ctrl-" + str + if event.state & gtk.gdk.MOD1_MASK: + str = "alt-" + str + if event.state & gtk.gdk.SHIFT_MASK: + str = "shift-" + str + + return str + + +class StepMakerWindow: + + # Constructor + + def __init__(self): + # Window + self.window = gtk.Window(gtk.WINDOW_TOPLEVEL) + self.window.set_title("Step Maker Window") + self.window.connect("delete-event", self.delete_event) + self.window.connect("destroy", self.destroy) + self.window.set_default_size(600, 800) + + # Main box (inside a frame which is inside a VBox) + self.menu_vbox = gtk.VBox() + self.window.add(self.menu_vbox) + self.menu_vbox.show() + + frame = gtk.Frame() + frame.set_border_width(10) + frame.set_shadow_type(gtk.SHADOW_NONE) + self.menu_vbox.pack_end(frame) + frame.show() + + self.main_vbox = gtk.VBox(spacing=10) + frame.add(self.main_vbox) + self.main_vbox.show() + + # EventBox + self.scrolledwindow = gtk.ScrolledWindow() + self.scrolledwindow.set_policy(gtk.POLICY_AUTOMATIC, + gtk.POLICY_AUTOMATIC) + self.scrolledwindow.set_shadow_type(gtk.SHADOW_NONE) + self.main_vbox.pack_start(self.scrolledwindow) + self.scrolledwindow.show() + + table = gtk.Table(1, 1) + self.scrolledwindow.add_with_viewport(table) + table.show() + table.realize() + + self.event_box = gtk.EventBox() + table.attach(self.event_box, 0, 1, 0, 1, gtk.EXPAND, gtk.EXPAND) + self.event_box.show() + self.event_box.realize() + + # Image + self.image = gtk.Image() + self.event_box.add(self.image) + self.image.show() + + # Data VBox + self.data_vbox = gtk.VBox(spacing=10) + self.main_vbox.pack_start(self.data_vbox, expand=False) + self.data_vbox.show() + + # User VBox + self.user_vbox = gtk.VBox(spacing=10) + self.main_vbox.pack_start(self.user_vbox, expand=False) + self.user_vbox.show() + + # Screendump ID HBox + box = gtk.HBox(spacing=10) + self.data_vbox.pack_start(box) + box.show() + + label = gtk.Label("Screendump ID:") + box.pack_start(label, False) + label.show() + + self.entry_screendump = gtk.Entry() + self.entry_screendump.set_editable(False) + box.pack_start(self.entry_screendump) + self.entry_screendump.show() + + label = gtk.Label("Time:") + box.pack_start(label, False) + label.show() + + self.entry_time = gtk.Entry() + self.entry_time.set_editable(False) + self.entry_time.set_width_chars(10) + box.pack_start(self.entry_time, False) + self.entry_time.show() + + # Comment HBox + box = gtk.HBox(spacing=10) + self.data_vbox.pack_start(box) + box.show() + + label = gtk.Label("Comment:") + box.pack_start(label, False) + label.show() + + self.entry_comment = gtk.Entry() + box.pack_start(self.entry_comment) + self.entry_comment.show() + + # Sleep HBox + box = gtk.HBox(spacing=10) + self.data_vbox.pack_start(box) + box.show() + + self.check_sleep = gtk.CheckButton("Sleep:") + self.check_sleep.connect("toggled", self.event_check_sleep_toggled) + box.pack_start(self.check_sleep, False) + self.check_sleep.show() + + self.spin_sleep = gtk.SpinButton(gtk.Adjustment(0, 0, 50000, 1, 10, 0), + climb_rate=0.0) + box.pack_start(self.spin_sleep, False) + self.spin_sleep.show() + + # Barrier HBox + box = gtk.HBox(spacing=10) + self.data_vbox.pack_start(box) + box.show() + + self.check_barrier = gtk.CheckButton("Barrier:") + self.check_barrier.connect("toggled", self.event_check_barrier_toggled) + box.pack_start(self.check_barrier, False) + self.check_barrier.show() + + vbox = gtk.VBox() + box.pack_start(vbox) + vbox.show() + + self.label_barrier_region = gtk.Label("Region:") + self.label_barrier_region.set_alignment(0, 0.5) + vbox.pack_start(self.label_barrier_region) + self.label_barrier_region.show() + + self.label_barrier_md5sum = gtk.Label("MD5:") + self.label_barrier_md5sum.set_alignment(0, 0.5) + vbox.pack_start(self.label_barrier_md5sum) + self.label_barrier_md5sum.show() + + self.label_barrier_timeout = gtk.Label("Timeout:") + box.pack_start(self.label_barrier_timeout, False) + self.label_barrier_timeout.show() + + self.spin_barrier_timeout = gtk.SpinButton(gtk.Adjustment(0, 0, 50000, + 1, 10, 0), + climb_rate=0.0) + box.pack_start(self.spin_barrier_timeout, False) + self.spin_barrier_timeout.show() + + self.check_barrier_optional = gtk.CheckButton("Optional") + box.pack_start(self.check_barrier_optional, False) + self.check_barrier_optional.show() + + # Keystrokes HBox + box = gtk.HBox(spacing=10) + self.data_vbox.pack_start(box) + box.show() + + label = gtk.Label("Keystrokes:") + box.pack_start(label, False) + label.show() + + frame = gtk.Frame() + frame.set_shadow_type(gtk.SHADOW_IN) + box.pack_start(frame) + frame.show() + + self.text_buffer = gtk.TextBuffer() + self.entry_keys = gtk.TextView(self.text_buffer) + self.entry_keys.set_wrap_mode(gtk.WRAP_WORD) + self.entry_keys.connect("key-press-event", self.event_key_press) + frame.add(self.entry_keys) + self.entry_keys.show() + + self.check_manual = gtk.CheckButton("Manual") + self.check_manual.connect("toggled", self.event_manual_toggled) + box.pack_start(self.check_manual, False) + self.check_manual.show() + + button = gtk.Button("Clear") + button.connect("clicked", self.event_clear_clicked) + box.pack_start(button, False) + button.show() + + # Mouse click HBox + box = gtk.HBox(spacing=10) + self.data_vbox.pack_start(box) + box.show() + + label = gtk.Label("Mouse action:") + box.pack_start(label, False) + label.show() + + self.button_capture = gtk.Button("Capture") + box.pack_start(self.button_capture, False) + self.button_capture.show() + + self.check_mousemove = gtk.CheckButton("Move: ...") + box.pack_start(self.check_mousemove, False) + self.check_mousemove.show() + + self.check_mouseclick = gtk.CheckButton("Click: ...") + box.pack_start(self.check_mouseclick, False) + self.check_mouseclick.show() + + self.spin_sensitivity = gtk.SpinButton(gtk.Adjustment(1, 1, 100, 1, 10, + 0), + climb_rate=0.0) + box.pack_end(self.spin_sensitivity, False) + self.spin_sensitivity.show() + + label = gtk.Label("Sensitivity:") + box.pack_end(label, False) + label.show() + + self.spin_latency = gtk.SpinButton(gtk.Adjustment(10, 1, 500, 1, 10, 0), + climb_rate=0.0) + box.pack_end(self.spin_latency, False) + self.spin_latency.show() + + label = gtk.Label("Latency:") + box.pack_end(label, False) + label.show() + + self.handler_event_box_press = None + self.handler_event_box_release = None + self.handler_event_box_scroll = None + self.handler_event_box_motion = None + self.handler_event_box_expose = None + + self.window.realize() + self.window.show() + + self.clear_state() + + # Utilities + + def message(self, text, title): + dlg = gtk.MessageDialog(self.window, + gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT, + gtk.MESSAGE_INFO, + gtk.BUTTONS_CLOSE, + title) + dlg.set_title(title) + dlg.format_secondary_text(text) + response = dlg.run() + dlg.destroy() + + + def question_yes_no(self, text, title): + dlg = gtk.MessageDialog(self.window, + gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT, + gtk.MESSAGE_QUESTION, + gtk.BUTTONS_YES_NO, + title) + dlg.set_title(title) + dlg.format_secondary_text(text) + response = dlg.run() + dlg.destroy() + if response == gtk.RESPONSE_YES: + return True + return False + + + def inputdialog(self, text, title, default_response=""): + # Define a little helper function + def inputdialog_entry_activated(entry): + dlg.response(gtk.RESPONSE_OK) + + # Create the dialog + dlg = gtk.MessageDialog(self.window, + gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT, + gtk.MESSAGE_QUESTION, + gtk.BUTTONS_OK_CANCEL, + title) + dlg.set_title(title) + dlg.format_secondary_text(text) + + # Create an entry widget + entry = gtk.Entry() + entry.set_text(default_response) + entry.connect("activate", inputdialog_entry_activated) + dlg.vbox.pack_start(entry) + entry.show() + + # Run the dialog + response = dlg.run() + dlg.destroy() + if response == gtk.RESPONSE_OK: + return entry.get_text() + return None + + + def filedialog(self, title=None, default_filename=None): + chooser = gtk.FileChooserDialog(title=title, parent=self.window, + action=gtk.FILE_CHOOSER_ACTION_OPEN, + buttons=(gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL, gtk.STOCK_OPEN, + gtk.RESPONSE_OK)) + chooser.resize(700, 500) + if default_filename: + chooser.set_filename(os.path.abspath(default_filename)) + filename = None + response = chooser.run() + if response == gtk.RESPONSE_OK: + filename = chooser.get_filename() + chooser.destroy() + return filename + + + def redirect_event_box_input(self, press=None, release=None, scroll=None, + motion=None, expose=None): + if self.handler_event_box_press != None: \ + self.event_box.disconnect(self.handler_event_box_press) + if self.handler_event_box_release != None: \ + self.event_box.disconnect(self.handler_event_box_release) + if self.handler_event_box_scroll != None: \ + self.event_box.disconnect(self.handler_event_box_scroll) + if self.handler_event_box_motion != None: \ + self.event_box.disconnect(self.handler_event_box_motion) + if self.handler_event_box_expose != None: \ + self.event_box.disconnect(self.handler_event_box_expose) + self.handler_event_box_press = None + self.handler_event_box_release = None + self.handler_event_box_scroll = None + self.handler_event_box_motion = None + self.handler_event_box_expose = None + if press != None: self.handler_event_box_press = \ + self.event_box.connect("button-press-event", press) + if release != None: self.handler_event_box_release = \ + self.event_box.connect("button-release-event", release) + if scroll != None: self.handler_event_box_scroll = \ + self.event_box.connect("scroll-event", scroll) + if motion != None: self.handler_event_box_motion = \ + self.event_box.connect("motion-notify-event", motion) + if expose != None: self.handler_event_box_expose = \ + self.event_box.connect_after("expose-event", expose) + + + def get_keys(self): + return self.text_buffer.get_text( + self.text_buffer.get_start_iter(), + self.text_buffer.get_end_iter()) + + + def add_key(self, key): + text = self.get_keys() + if len(text) > 0 and text[-1] != ' ': + text += " " + text += key + self.text_buffer.set_text(text) + + + def clear_keys(self): + self.text_buffer.set_text("") + + + def update_barrier_info(self): + if self.barrier_selected: + self.label_barrier_region.set_text("Selected region: Corner: " + \ + str(tuple(self.barrier_corner)) + \ + " Size: " + \ + str(tuple(self.barrier_size))) + else: + self.label_barrier_region.set_text("No region selected.") + self.label_barrier_md5sum.set_text("MD5: " + self.barrier_md5sum) + + + def update_mouse_click_info(self): + if self.mouse_click_captured: + self.check_mousemove.set_label("Move: " + \ + str(tuple(self.mouse_click_coords))) + self.check_mouseclick.set_label("Click: button %d" % + self.mouse_click_button) + else: + self.check_mousemove.set_label("Move: ...") + self.check_mouseclick.set_label("Click: ...") + + + def clear_state(self, clear_screendump=True): + # Recording time + self.entry_time.set_text("unknown") + if clear_screendump: + # Screendump + self.clear_image() + # Screendump ID + self.entry_screendump.set_text("") + # Comment + self.entry_comment.set_text("") + # Sleep + self.check_sleep.set_active(True) + self.check_sleep.set_active(False) + self.spin_sleep.set_value(10) + # Barrier + self.clear_barrier_state() + # Keystrokes + self.check_manual.set_active(False) + self.clear_keys() + # Mouse actions + self.check_mousemove.set_sensitive(False) + self.check_mouseclick.set_sensitive(False) + self.check_mousemove.set_active(False) + self.check_mouseclick.set_active(False) + self.mouse_click_captured = False + self.mouse_click_coords = [0, 0] + self.mouse_click_button = 0 + self.update_mouse_click_info() + + + def clear_barrier_state(self): + self.check_barrier.set_active(True) + self.check_barrier.set_active(False) + self.check_barrier_optional.set_active(False) + self.spin_barrier_timeout.set_value(10) + self.barrier_selection_started = False + self.barrier_selected = False + self.barrier_corner0 = [0, 0] + self.barrier_corner1 = [0, 0] + self.barrier_corner = [0, 0] + self.barrier_size = [0, 0] + self.barrier_md5sum = "" + self.update_barrier_info() + + + def set_image(self, w, h, data): + (self.image_width, self.image_height, self.image_data) = (w, h, data) + self.image.set_from_pixbuf(gtk.gdk.pixbuf_new_from_data( + data, gtk.gdk.COLORSPACE_RGB, False, 8, + w, h, w*3)) + hscrollbar = self.scrolledwindow.get_hscrollbar() + hscrollbar.set_range(0, w) + vscrollbar = self.scrolledwindow.get_vscrollbar() + vscrollbar.set_range(0, h) + + + def set_image_from_file(self, filename): + if not ppm_utils.image_verify_ppm_file(filename): + logging.warning("set_image_from_file: Warning: received invalid" + "screendump file") + return self.clear_image() + (w, h, data) = ppm_utils.image_read_from_ppm_file(filename) + self.set_image(w, h, data) + + + def clear_image(self): + self.image.clear() + self.image_width = 0 + self.image_height = 0 + self.image_data = "" + + + def update_screendump_id(self, data_dir): + if not self.image_data: + return + # Find a proper ID for the screendump + scrdump_md5sum = ppm_utils.image_md5sum(self.image_width, + self.image_height, + self.image_data) + scrdump_id = ppm_utils.find_id_for_screendump(scrdump_md5sum, data_dir) + if not scrdump_id: + # Not found; generate one + scrdump_id = ppm_utils.generate_id_for_screendump(scrdump_md5sum, + data_dir) + self.entry_screendump.set_text(scrdump_id) + + + def get_step_lines(self, data_dir=None): + if self.check_barrier.get_active() and not self.barrier_selected: + self.message("No barrier region selected.", "Error") + return + + str = "step" + + # Add step recording time + if self.entry_time.get_text(): + str += " " + self.entry_time.get_text() + + str += "\n" + + # Add screendump line + if self.image_data: + str += "screendump %s\n" % self.entry_screendump.get_text() + + # Add comment + if self.entry_comment.get_text(): + str += "# %s\n" % self.entry_comment.get_text() + + # Add sleep line + if self.check_sleep.get_active(): + str += "sleep %d\n" % self.spin_sleep.get_value() + + # Add barrier_2 line + if self.check_barrier.get_active(): + str += "barrier_2 %d %d %d %d %s %d" % ( + self.barrier_size[0], self.barrier_size[1], + self.barrier_corner[0], self.barrier_corner[1], + self.barrier_md5sum, self.spin_barrier_timeout.get_value()) + if self.check_barrier_optional.get_active(): + str += " optional" + str += "\n" + + # Add "Sending keys" comment + keys_to_send = self.get_keys().split() + if keys_to_send: + str += "# Sending keys: %s\n" % self.get_keys() + + # Add key and var lines + for key in keys_to_send: + if key.startswith("$"): + varname = key[1:] + str += "var %s\n" % varname + else: + str += "key %s\n" % key + + # Add mousemove line + if self.check_mousemove.get_active(): + str += "mousemove %d %d\n" % (self.mouse_click_coords[0], + self.mouse_click_coords[1]) + + # Add mouseclick line + if self.check_mouseclick.get_active(): + dict = { 1 : 1, + 2 : 2, + 3 : 4 } + str += "mouseclick %d\n" % dict[self.mouse_click_button] + + # Write screendump and cropped screendump image files + if data_dir and self.image_data: + # Create the data dir if it doesn't exist + if not os.path.exists(data_dir): + os.makedirs(data_dir) + # Get the full screendump filename + scrdump_filename = os.path.join(data_dir, + self.entry_screendump.get_text()) + # Write screendump file if it doesn't exist + if not os.path.exists(scrdump_filename): + try: + ppm_utils.image_write_to_ppm_file(scrdump_filename, + self.image_width, + self.image_height, + self.image_data) + except IOError: + self.message("Could not write screendump file.", "Error") + + #if self.check_barrier.get_active(): + # # Crop image to get the cropped screendump + # (cw, ch, cdata) = ppm_utils.image_crop( + # self.image_width, self.image_height, self.image_data, + # self.barrier_corner[0], self.barrier_corner[1], + # self.barrier_size[0], self.barrier_size[1]) + # cropped_scrdump_md5sum = ppm_utils.image_md5sum(cw, ch, cdata) + # cropped_scrdump_filename = \ + # ppm_utils.get_cropped_screendump_filename(scrdump_filename, + # cropped_scrdump_md5sum) + # # Write cropped screendump file + # try: + # ppm_utils.image_write_to_ppm_file(cropped_scrdump_filename, + # cw, ch, cdata) + # except IOError: + # self.message("Could not write cropped screendump file.", + # "Error") + + return str + + def set_state_from_step_lines(self, str, data_dir, warn=True): + self.clear_state() + + for line in str.splitlines(): + words = line.split() + if not words: + continue + + if line.startswith("#") \ + and not self.entry_comment.get_text() \ + and not line.startswith("# Sending keys:") \ + and not line.startswith("# ----"): + self.entry_comment.set_text(line.strip("#").strip()) + + elif words[0] == "step": + if len(words) >= 2: + self.entry_time.set_text(words[1]) + + elif words[0] == "screendump": + self.entry_screendump.set_text(words[1]) + self.set_image_from_file(os.path.join(data_dir, words[1])) + + elif words[0] == "sleep": + self.spin_sleep.set_value(int(words[1])) + self.check_sleep.set_active(True) + + elif words[0] == "key": + self.add_key(words[1]) + + elif words[0] == "var": + self.add_key("$%s" % words[1]) + + elif words[0] == "mousemove": + self.mouse_click_captured = True + self.mouse_click_coords = [int(words[1]), int(words[2])] + self.update_mouse_click_info() + + elif words[0] == "mouseclick": + self.mouse_click_captured = True + self.mouse_click_button = int(words[1]) + self.update_mouse_click_info() + + elif words[0] == "barrier_2": + # Get region corner and size from step lines + self.barrier_corner = [int(words[3]), int(words[4])] + self.barrier_size = [int(words[1]), int(words[2])] + # Get corner0 and corner1 from step lines + self.barrier_corner0 = self.barrier_corner + self.barrier_corner1 = [self.barrier_corner[0] + + self.barrier_size[0] - 1, + self.barrier_corner[1] + + self.barrier_size[1] - 1] + # Get the md5sum + self.barrier_md5sum = words[5] + # Pretend the user selected the region with the mouse + self.barrier_selection_started = True + self.barrier_selected = True + # Update label widgets according to region information + self.update_barrier_info() + # Check the barrier checkbutton + self.check_barrier.set_active(True) + # Set timeout value + self.spin_barrier_timeout.set_value(int(words[6])) + # Set 'optional' checkbutton state + self.check_barrier_optional.set_active(words[-1] == "optional") + # Update the image widget + self.event_box.queue_draw() + + if warn: + # See if the computed md5sum matches the one recorded in + # the file + computed_md5sum = ppm_utils.get_region_md5sum( + self.image_width, self.image_height, + self.image_data, self.barrier_corner[0], + self.barrier_corner[1], self.barrier_size[0], + self.barrier_size[1]) + if computed_md5sum != self.barrier_md5sum: + self.message("Computed MD5 sum (%s) differs from MD5" + " sum recorded in steps file (%s)" % + (computed_md5sum, self.barrier_md5sum), + "Warning") + + # Events + + def delete_event(self, widget, event): + pass + + def destroy(self, widget): + gtk.main_quit() + + def event_check_barrier_toggled(self, widget): + if self.check_barrier.get_active(): + self.redirect_event_box_input( + self.event_button_press, + self.event_button_release, + None, + None, + self.event_expose) + self.event_box.queue_draw() + self.event_box.window.set_cursor(gtk.gdk.Cursor(gtk.gdk.CROSSHAIR)) + self.label_barrier_region.set_sensitive(True) + self.label_barrier_md5sum.set_sensitive(True) + self.label_barrier_timeout.set_sensitive(True) + self.spin_barrier_timeout.set_sensitive(True) + self.check_barrier_optional.set_sensitive(True) + else: + self.redirect_event_box_input() + self.event_box.queue_draw() + self.event_box.window.set_cursor(None) + self.label_barrier_region.set_sensitive(False) + self.label_barrier_md5sum.set_sensitive(False) + self.label_barrier_timeout.set_sensitive(False) + self.spin_barrier_timeout.set_sensitive(False) + self.check_barrier_optional.set_sensitive(False) + + def event_check_sleep_toggled(self, widget): + if self.check_sleep.get_active(): + self.spin_sleep.set_sensitive(True) + else: + self.spin_sleep.set_sensitive(False) + + def event_manual_toggled(self, widget): + self.entry_keys.grab_focus() + + def event_clear_clicked(self, widget): + self.clear_keys() + self.entry_keys.grab_focus() + + def event_expose(self, widget, event): + if not self.barrier_selection_started: + return + (corner, size) = corner_and_size_clipped(self.barrier_corner0, + self.barrier_corner1, + self.event_box.size_request()) + gc = self.event_box.window.new_gc(line_style=gtk.gdk.LINE_DOUBLE_DASH, + line_width=1) + gc.set_foreground(gc.get_colormap().alloc_color("red")) + gc.set_background(gc.get_colormap().alloc_color("dark red")) + gc.set_dashes(0, (4, 4)) + self.event_box.window.draw_rectangle( + gc, False, + corner[0], corner[1], + size[0]-1, size[1]-1) + + def event_drag_motion(self, widget, event): + old_corner1 = self.barrier_corner1 + self.barrier_corner1 = [int(event.x), int(event.y)] + (corner, size) = corner_and_size_clipped(self.barrier_corner0, + self.barrier_corner1, + self.event_box.size_request()) + (old_corner, old_size) = corner_and_size_clipped(self.barrier_corner0, + old_corner1, + self.event_box.size_request()) + corner0 = [min(corner[0], old_corner[0]), min(corner[1], old_corner[1])] + corner1 = [max(corner[0] + size[0], old_corner[0] + old_size[0]), + max(corner[1] + size[1], old_corner[1] + old_size[1])] + size = [corner1[0] - corner0[0] + 1, + corner1[1] - corner0[1] + 1] + self.event_box.queue_draw_area(corner0[0], corner0[1], size[0], size[1]) + + def event_button_press(self, widget, event): + (corner, size) = corner_and_size_clipped(self.barrier_corner0, + self.barrier_corner1, + self.event_box.size_request()) + self.event_box.queue_draw_area(corner[0], corner[1], size[0], size[1]) + self.barrier_corner0 = [int(event.x), int(event.y)] + self.barrier_corner1 = [int(event.x), int(event.y)] + self.redirect_event_box_input( + self.event_button_press, + self.event_button_release, + None, + self.event_drag_motion, + self.event_expose) + self.barrier_selection_started = True + + def event_button_release(self, widget, event): + self.redirect_event_box_input( + self.event_button_press, + self.event_button_release, + None, + None, + self.event_expose) + (self.barrier_corner, self.barrier_size) = \ + corner_and_size_clipped(self.barrier_corner0, self.barrier_corner1, + self.event_box.size_request()) + self.barrier_md5sum = ppm_utils.get_region_md5sum( + self.image_width, self.image_height, self.image_data, + self.barrier_corner[0], self.barrier_corner[1], + self.barrier_size[0], self.barrier_size[1]) + self.barrier_selected = True + self.update_barrier_info() + + def event_key_press(self, widget, event): + if self.check_manual.get_active(): + return False + str = key_event_to_qemu_string(event) + self.add_key(str) + return True + + +class StepEditor(StepMakerWindow): + ui = ''' + + + + + + + + + + + + + + + + + + + + + +''' + + # Constructor + + def __init__(self, filename=None): + StepMakerWindow.__init__(self) + + self.steps_filename = None + self.steps = [] + + # Create a UIManager instance + uimanager = gtk.UIManager() + + # Add the accelerator group to the toplevel window + accelgroup = uimanager.get_accel_group() + self.window.add_accel_group(accelgroup) + + # Create an ActionGroup + actiongroup = gtk.ActionGroup('StepEditor') + + # Create actions + actiongroup.add_actions([ + ('Quit', gtk.STOCK_QUIT, '_Quit', None, 'Quit the Program', + self.quit), + ('Open', gtk.STOCK_OPEN, '_Open', None, 'Open steps file', + self.open_steps_file), + ('CopyStep', gtk.STOCK_COPY, '_Copy current step...', "", + 'Copy current step to user specified position', self.copy_step), + ('DeleteStep', gtk.STOCK_DELETE, '_Delete current step', "", + 'Delete current step', self.event_remove_clicked), + ('InsertNewBefore', gtk.STOCK_ADD, '_New step before current', "", + 'Insert new step before current step', self.insert_before), + ('InsertNewAfter', gtk.STOCK_ADD, 'N_ew step after current', "", + 'Insert new step after current step', self.insert_after), + ('InsertStepsBefore', gtk.STOCK_ADD, '_Steps before current...', + "", 'Insert steps (from file) before current step', + self.insert_steps_before), + ('InsertStepsAfter', gtk.STOCK_ADD, 'Steps _after current...', "", + 'Insert steps (from file) after current step', + self.insert_steps_after), + ('CleanUp', gtk.STOCK_DELETE, '_Clean up data directory', "", + 'Move unused PPM files to a backup directory', self.cleanup), + ('File', None, '_File'), + ('Edit', None, '_Edit'), + ('Insert', None, '_Insert'), + ('Tools', None, '_Tools') + ]) + + def create_shortcut(name, callback, keyname): + # Create an action + action = gtk.Action(name, None, None, None) + # Connect a callback to the action + action.connect("activate", callback) + actiongroup.add_action_with_accel(action, keyname) + # Have the action use accelgroup + action.set_accel_group(accelgroup) + # Connect the accelerator to the action + action.connect_accelerator() + + create_shortcut("Next", self.event_next_clicked, "Page_Down") + create_shortcut("Previous", self.event_prev_clicked, "Page_Up") + + # Add the actiongroup to the uimanager + uimanager.insert_action_group(actiongroup, 0) + + # Add a UI description + uimanager.add_ui_from_string(self.ui) + + # Create a MenuBar + menubar = uimanager.get_widget('/MenuBar') + self.menu_vbox.pack_start(menubar, False) + + # Remember the Edit menu bar for future reference + self.menu_edit = uimanager.get_widget('/MenuBar/Edit') + self.menu_edit.set_sensitive(False) + + # Remember the Insert menu bar for future reference + self.menu_insert = uimanager.get_widget('/MenuBar/Insert') + self.menu_insert.set_sensitive(False) + + # Remember the Tools menu bar for future reference + self.menu_tools = uimanager.get_widget('/MenuBar/Tools') + self.menu_tools.set_sensitive(False) + + # Next/Previous HBox + hbox = gtk.HBox(spacing=10) + self.user_vbox.pack_start(hbox) + hbox.show() + + self.button_first = gtk.Button(stock=gtk.STOCK_GOTO_FIRST) + self.button_first.connect("clicked", self.event_first_clicked) + hbox.pack_start(self.button_first) + self.button_first.show() + + #self.button_prev = gtk.Button("<< Previous") + self.button_prev = gtk.Button(stock=gtk.STOCK_GO_BACK) + self.button_prev.connect("clicked", self.event_prev_clicked) + hbox.pack_start(self.button_prev) + self.button_prev.show() + + self.label_step = gtk.Label("Step:") + hbox.pack_start(self.label_step, False) + self.label_step.show() + + self.entry_step_num = gtk.Entry() + self.entry_step_num.connect("activate", self.event_entry_step_activated) + self.entry_step_num.set_width_chars(3) + hbox.pack_start(self.entry_step_num, False) + self.entry_step_num.show() + + #self.button_next = gtk.Button("Next >>") + self.button_next = gtk.Button(stock=gtk.STOCK_GO_FORWARD) + self.button_next.connect("clicked", self.event_next_clicked) + hbox.pack_start(self.button_next) + self.button_next.show() + + self.button_last = gtk.Button(stock=gtk.STOCK_GOTO_LAST) + self.button_last.connect("clicked", self.event_last_clicked) + hbox.pack_start(self.button_last) + self.button_last.show() + + # Save HBox + hbox = gtk.HBox(spacing=10) + self.user_vbox.pack_start(hbox) + hbox.show() + + self.button_save = gtk.Button("_Save current step") + self.button_save.connect("clicked", self.event_save_clicked) + hbox.pack_start(self.button_save) + self.button_save.show() + + self.button_remove = gtk.Button("_Delete current step") + self.button_remove.connect("clicked", self.event_remove_clicked) + hbox.pack_start(self.button_remove) + self.button_remove.show() + + self.button_replace = gtk.Button("_Replace screendump") + self.button_replace.connect("clicked", self.event_replace_clicked) + hbox.pack_start(self.button_replace) + self.button_replace.show() + + # Disable unused widgets + self.button_capture.set_sensitive(False) + self.spin_latency.set_sensitive(False) + self.spin_sensitivity.set_sensitive(False) + + # Disable main vbox because no steps file is loaded + self.main_vbox.set_sensitive(False) + + # Set title + self.window.set_title("Step Editor") + + # Events + + def delete_event(self, widget, event): + # Make sure the step is saved (if the user wants it to be) + self.verify_save() + + def event_first_clicked(self, widget): + if not self.steps: + return + # Make sure the step is saved (if the user wants it to be) + self.verify_save() + # Go to first step + self.set_step(0) + + def event_last_clicked(self, widget): + if not self.steps: + return + # Make sure the step is saved (if the user wants it to be) + self.verify_save() + # Go to last step + self.set_step(len(self.steps) - 1) + + def event_prev_clicked(self, widget): + if not self.steps: + return + # Make sure the step is saved (if the user wants it to be) + self.verify_save() + # Go to previous step + index = self.current_step_index - 1 + if self.steps: + index = index % len(self.steps) + self.set_step(index) + + def event_next_clicked(self, widget): + if not self.steps: + return + # Make sure the step is saved (if the user wants it to be) + self.verify_save() + # Go to next step + index = self.current_step_index + 1 + if self.steps: + index = index % len(self.steps) + self.set_step(index) + + def event_entry_step_activated(self, widget): + if not self.steps: + return + step_index = self.entry_step_num.get_text() + if not step_index.isdigit(): + return + step_index = int(step_index) - 1 + if step_index == self.current_step_index: + return + self.verify_save() + self.set_step(step_index) + + def event_save_clicked(self, widget): + if not self.steps: + return + self.save_step() + + def event_remove_clicked(self, widget): + if not self.steps: + return + if not self.question_yes_no("This will modify the steps file." + " Are you sure?", "Remove step?"): + return + # Remove step + del self.steps[self.current_step_index] + # Write changes to file + self.write_steps_file(self.steps_filename) + # Move to previous step + self.set_step(self.current_step_index) + + def event_replace_clicked(self, widget): + if not self.steps: + return + # Let the user choose a screendump file + current_filename = os.path.join(self.steps_data_dir, + self.entry_screendump.get_text()) + filename = self.filedialog("Choose PPM image file", + default_filename=current_filename) + if not filename: + return + if not ppm_utils.image_verify_ppm_file(filename): + self.message("Not a valid PPM image file.", "Error") + return + self.clear_image() + self.clear_barrier_state() + self.set_image_from_file(filename) + self.update_screendump_id(self.steps_data_dir) + + # Menu actions + + def open_steps_file(self, action): + # Make sure the step is saved (if the user wants it to be) + self.verify_save() + # Let the user choose a steps file + current_filename = self.steps_filename + filename = self.filedialog("Open steps file", + default_filename=current_filename) + if not filename: + return + self.set_steps_file(filename) + + def quit(self, action): + # Make sure the step is saved (if the user wants it to be) + self.verify_save() + # Quit + gtk.main_quit() + + def copy_step(self, action): + if not self.steps: + return + self.verify_save() + self.set_step(self.current_step_index) + # Get the desired position + step_index = self.inputdialog("Copy step to position:", + "Copy step", + str(self.current_step_index + 2)) + if not step_index: + return + step_index = int(step_index) - 1 + # Get the lines of the current step + step = self.steps[self.current_step_index] + # Insert new step at position step_index + self.steps.insert(step_index, step) + # Go to new step + self.set_step(step_index) + # Write changes to disk + self.write_steps_file(self.steps_filename) + + def insert_before(self, action): + if not self.steps_filename: + return + if not self.question_yes_no("This will modify the steps file." + " Are you sure?", "Insert new step?"): + return + self.verify_save() + step_index = self.current_step_index + # Get the lines of a blank step + self.clear_state() + step = self.get_step_lines() + # Insert new step at position step_index + self.steps.insert(step_index, step) + # Go to new step + self.set_step(step_index) + # Write changes to disk + self.write_steps_file(self.steps_filename) + + def insert_after(self, action): + if not self.steps_filename: + return + if not self.question_yes_no("This will modify the steps file." + " Are you sure?", "Insert new step?"): + return + self.verify_save() + step_index = self.current_step_index + 1 + # Get the lines of a blank step + self.clear_state() + step = self.get_step_lines() + # Insert new step at position step_index + self.steps.insert(step_index, step) + # Go to new step + self.set_step(step_index) + # Write changes to disk + self.write_steps_file(self.steps_filename) + + def insert_steps(self, filename, index): + # Read the steps file + (steps, header) = self.read_steps_file(filename) + + data_dir = ppm_utils.get_data_dir(filename) + for step in steps: + self.set_state_from_step_lines(step, data_dir, warn=False) + step = self.get_step_lines(self.steps_data_dir) + + # Insert steps into self.steps + self.steps[index:index] = steps + # Write changes to disk + self.write_steps_file(self.steps_filename) + + def insert_steps_before(self, action): + if not self.steps_filename: + return + # Let the user choose a steps file + current_filename = self.steps_filename + filename = self.filedialog("Choose steps file", + default_filename=current_filename) + if not filename: + return + self.verify_save() + + step_index = self.current_step_index + # Insert steps at position step_index + self.insert_steps(filename, step_index) + # Go to new steps + self.set_step(step_index) + + def insert_steps_after(self, action): + if not self.steps_filename: + return + # Let the user choose a steps file + current_filename = self.steps_filename + filename = self.filedialog("Choose steps file", + default_filename=current_filename) + if not filename: + return + self.verify_save() + + step_index = self.current_step_index + 1 + # Insert new steps at position step_index + self.insert_steps(filename, step_index) + # Go to new steps + self.set_step(step_index) + + def cleanup(self, action): + if not self.steps_filename: + return + if not self.question_yes_no("All unused PPM files will be moved to a" + " backup directory. Are you sure?", + "Clean up data directory?"): + return + # Remember the current step index + current_step_index = self.current_step_index + # Get the backup dir + backup_dir = os.path.join(self.steps_data_dir, "backup") + # Create it if it doesn't exist + if not os.path.exists(backup_dir): + os.makedirs(backup_dir) + # Move all files to the backup dir + for filename in glob.glob(os.path.join(self.steps_data_dir, + "*.[Pp][Pp][Mm]")): + shutil.move(filename, backup_dir) + # Get the used files back + for step in self.steps: + self.set_state_from_step_lines(step, backup_dir, warn=False) + self.get_step_lines(self.steps_data_dir) + # Remove the used files from the backup dir + used_files = os.listdir(self.steps_data_dir) + for filename in os.listdir(backup_dir): + if filename in used_files: + os.unlink(os.path.join(backup_dir, filename)) + # Restore step index + self.set_step(current_step_index) + # Inform the user + self.message("All unused PPM files may be found at %s." % + os.path.abspath(backup_dir), + "Clean up data directory") + + # Methods + + def read_steps_file(self, filename): + steps = [] + header = "" + + file = open(filename, "r") + for line in file.readlines(): + words = line.split() + if not words: + continue + if line.startswith("# ----"): + continue + if words[0] == "step": + steps.append("") + if steps: + steps[-1] += line + else: + header += line + file.close() + + return (steps, header) + + def set_steps_file(self, filename): + try: + (self.steps, self.header) = self.read_steps_file(filename) + except (TypeError, IOError): + self.message("Cannot read file %s." % filename, "Error") + return + + self.steps_filename = filename + self.steps_data_dir = ppm_utils.get_data_dir(filename) + # Go to step 0 + self.set_step(0) + + def set_step(self, index): + # Limit index to legal boundaries + if index < 0: + index = 0 + if index > len(self.steps) - 1: + index = len(self.steps) - 1 + + # Enable the menus + self.menu_edit.set_sensitive(True) + self.menu_insert.set_sensitive(True) + self.menu_tools.set_sensitive(True) + + # If no steps exist... + if self.steps == []: + self.current_step_index = index + self.current_step = None + # Set window title + self.window.set_title("Step Editor -- %s" % + os.path.basename(self.steps_filename)) + # Set step entry widget text + self.entry_step_num.set_text("") + # Clear the state of all widgets + self.clear_state() + # Disable the main vbox + self.main_vbox.set_sensitive(False) + return + + self.current_step_index = index + self.current_step = self.steps[index] + # Set window title + self.window.set_title("Step Editor -- %s -- step %d" % + (os.path.basename(self.steps_filename), + index + 1)) + # Set step entry widget text + self.entry_step_num.set_text(str(self.current_step_index + 1)) + # Load the state from the step lines + self.set_state_from_step_lines(self.current_step, self.steps_data_dir) + # Enable the main vbox + self.main_vbox.set_sensitive(True) + # Make sure the step lines in self.current_step are identical to the + # output of self.get_step_lines + self.current_step = self.get_step_lines() + + def verify_save(self): + if not self.steps: + return + # See if the user changed anything + if self.get_step_lines() != self.current_step: + if self.question_yes_no("Step contents have been modified." + " Save step?", "Save changes?"): + self.save_step() + + def save_step(self): + lines = self.get_step_lines(self.steps_data_dir) + if lines != None: + self.steps[self.current_step_index] = lines + self.current_step = lines + self.write_steps_file(self.steps_filename) + + def write_steps_file(self, filename): + file = open(filename, "w") + file.write(self.header) + for step in self.steps: + file.write("# " + "-" * 32 + "\n") + file.write(step) + file.close() + + +if __name__ == "__main__": + se = StepEditor() + if len(sys.argv) > 1: + se.set_steps_file(sys.argv[1]) + gtk.main() diff --git a/client/virt/virt_test_setup.py b/client/virt/virt_test_setup.py new file mode 100644 index 0000000..1125aea --- /dev/null +++ b/client/virt/virt_test_setup.py @@ -0,0 +1,700 @@ +""" +Library to perform pre/post test setup for KVM autotest. +""" +import os, shutil, tempfile, re, ConfigParser, glob, inspect +import logging, time +from autotest_lib.client.common_lib import error +from autotest_lib.client.bin import utils + + +@error.context_aware +def cleanup(dir): + """ + If dir is a mountpoint, do what is possible to unmount it. Afterwards, + try to remove it. + + @param dir: Directory to be cleaned up. + """ + error.context("cleaning up unattended install directory %s" % dir) + if os.path.ismount(dir): + utils.run('fuser -k %s' % dir, ignore_status=True) + utils.run('umount %s' % dir) + if os.path.isdir(dir): + shutil.rmtree(dir) + + +@error.context_aware +def clean_old_image(image): + """ + Clean a leftover image file from previous processes. If it contains a + mounted file system, do the proper cleanup procedures. + + @param image: Path to image to be cleaned up. + """ + error.context("cleaning up old leftover image %s" % image) + if os.path.exists(image): + mtab = open('/etc/mtab', 'r') + mtab_contents = mtab.read() + mtab.close() + if image in mtab_contents: + utils.run('fuser -k %s' % image, ignore_status=True) + utils.run('umount %s' % image) + os.remove(image) + + +def display_attributes(instance): + """ + Inspects a given class instance attributes and displays them, convenient + for debugging. + """ + logging.debug("Attributes set:") + for member in inspect.getmembers(instance): + name, value = member + attribute = getattr(instance, name) + if not (name.startswith("__") or callable(attribute) or not value): + logging.debug(" %s: %s", name, value) + + +class Disk(object): + """ + Abstract class for Disk objects, with the common methods implemented. + """ + def __init__(self): + self.path = None + + + def setup_answer_file(self, filename, contents): + utils.open_write_close(os.path.join(self.mount, filename), contents) + + + def copy_to(self, src): + logging.debug("Copying %s to disk image mount", src) + dst = os.path.join(self.mount, os.path.basename(src)) + if os.path.isdir(src): + shutil.copytree(src, dst) + elif os.path.isfile(src): + shutil.copyfile(src, dst) + + + def close(self): + os.chmod(self.path, 0755) + cleanup(self.mount) + logging.debug("Disk %s successfuly set", self.path) + + +class FloppyDisk(Disk): + """ + Represents a 1.44 MB floppy disk. We can copy files to it, and setup it in + convenient ways. + """ + @error.context_aware + def __init__(self, path, qemu_img_binary, tmpdir): + error.context("Creating unattended install floppy image %s" % path) + self.tmpdir = tmpdir + self.mount = tempfile.mkdtemp(prefix='floppy_', dir=self.tmpdir) + self.virtio_mount = None + self.path = path + clean_old_image(path) + if not os.path.isdir(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + + try: + c_cmd = '%s create -f raw %s 1440k' % (qemu_img_binary, path) + utils.run(c_cmd) + f_cmd = 'mkfs.msdos -s 1 %s' % path + utils.run(f_cmd) + m_cmd = 'mount -o loop,rw %s %s' % (path, self.mount) + utils.run(m_cmd) + except error.CmdError, e: + cleanup(self.mount) + raise + + + def _copy_virtio_drivers(self, virtio_floppy): + """ + Copy the virtio drivers on the virtio floppy to the install floppy. + + 1) Mount the floppy containing the viostor drivers + 2) Copy its contents to the root of the install floppy + """ + virtio_mount = tempfile.mkdtemp(prefix='virtio_floppy_', + dir=self.tmpdir) + + pwd = os.getcwd() + try: + m_cmd = 'mount -o loop %s %s' % (virtio_floppy, virtio_mount) + utils.run(m_cmd) + os.chdir(virtio_mount) + path_list = glob.glob('*') + for path in path_list: + self.copy_to(path) + finally: + os.chdir(pwd) + cleanup(virtio_mount) + + + def setup_virtio_win2003(self, virtio_floppy, virtio_oemsetup_id): + """ + Setup the install floppy with the virtio storage drivers, win2003 style. + + Win2003 and WinXP depend on the file txtsetup.oem file to install + the virtio drivers from the floppy, which is a .ini file. + Process: + + 1) Copy the virtio drivers on the virtio floppy to the install floppy + 2) Parse the ini file with config parser + 3) Modify the identifier of the default session that is going to be + executed on the config parser object + 4) Re-write the config file to the disk + """ + self._copy_virtio_drivers(virtio_floppy) + txtsetup_oem = os.path.join(self.mount, 'txtsetup.oem') + if not os.path.isfile(txtsetup_oem): + raise IOError('File txtsetup.oem not found on the install ' + 'floppy. Please verify if your floppy virtio ' + 'driver image has this file') + parser = ConfigParser.ConfigParser() + parser.read(txtsetup_oem) + if not parser.has_section('Defaults'): + raise ValueError('File txtsetup.oem does not have the session ' + '"Defaults". Please check txtsetup.oem') + default_driver = parser.get('Defaults', 'SCSI') + if default_driver != virtio_oemsetup_id: + parser.set('Defaults', 'SCSI', virtio_oemsetup_id) + fp = open(txtsetup_oem, 'w') + parser.write(fp) + fp.close() + + + def setup_virtio_win2008(self, virtio_floppy): + """ + Setup the install floppy with the virtio storage drivers, win2008 style. + + Win2008, Vista and 7 require people to point out the path to the drivers + on the unattended file, so we just need to copy the drivers to the + driver floppy disk. + Process: + + 1) Copy the virtio drivers on the virtio floppy to the install floppy + """ + self._copy_virtio_drivers(virtio_floppy) + + +class CdromDisk(Disk): + """ + Represents a CDROM disk that we can master according to our needs. + """ + def __init__(self, path, tmpdir): + self.mount = tempfile.mkdtemp(prefix='cdrom_unattended_', dir=tmpdir) + self.path = path + clean_old_image(path) + if not os.path.isdir(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + + + @error.context_aware + def close(self): + error.context("Creating unattended install CD image %s" % self.path) + g_cmd = ('mkisofs -o %s -max-iso9660-filenames ' + '-relaxed-filenames -D --input-charset iso8859-1 ' + '%s' % (self.path, self.mount)) + utils.run(g_cmd) + + os.chmod(self.path, 0755) + cleanup(self.mount) + logging.debug("unattended install CD image %s successfuly created", + self.path) + + +class UnattendedInstallConfig(object): + """ + Creates a floppy disk image that will contain a config file for unattended + OS install. The parameters to the script are retrieved from environment + variables. + """ + def __init__(self, test, params): + """ + Sets class atributes from test parameters. + + @param test: KVM test object. + @param params: Dictionary with test parameters. + """ + root_dir = test.bindir + images_dir = os.path.join(root_dir, 'images') + self.deps_dir = os.path.join(root_dir, 'deps') + self.unattended_dir = os.path.join(root_dir, 'unattended') + + attributes = ['kernel_args', 'finish_program', 'cdrom_cd1', + 'unattended_file', 'medium', 'url', 'kernel', 'initrd', + 'nfs_server', 'nfs_dir', 'install_virtio', 'floppy', + 'cdrom_unattended', 'boot_path', 'extra_params', + 'qemu_img_binary', 'cdkey', 'finish_program'] + + for a in attributes: + setattr(self, a, params.get(a, '')) + + if self.install_virtio == 'yes': + v_attributes = ['virtio_floppy', 'virtio_storage_path', + 'virtio_network_path', 'virtio_oemsetup_id', + 'virtio_network_installer'] + for va in v_attributes: + setattr(self, va, params.get(va, '')) + + self.tmpdir = test.tmpdir + + if getattr(self, 'unattended_file'): + self.unattended_file = os.path.join(root_dir, self.unattended_file) + + if getattr(self, 'finish_program'): + self.finish_program = os.path.join(root_dir, self.finish_program) + + if getattr(self, 'qemu_img_binary'): + if not os.path.isfile(getattr(self, 'qemu_img_binary')): + self.qemu_img_binary = os.path.join(root_dir, + self.qemu_img_binary) + + if getattr(self, 'cdrom_cd1'): + self.cdrom_cd1 = os.path.join(root_dir, self.cdrom_cd1) + self.cdrom_cd1_mount = tempfile.mkdtemp(prefix='cdrom_cd1_', + dir=self.tmpdir) + if self.medium == 'nfs': + self.nfs_mount = tempfile.mkdtemp(prefix='nfs_', + dir=self.tmpdir) + + if getattr(self, 'floppy'): + self.floppy = os.path.join(root_dir, self.floppy) + if not os.path.isdir(os.path.dirname(self.floppy)): + os.makedirs(os.path.dirname(self.floppy)) + + self.image_path = os.path.dirname(self.kernel) + + + @error.context_aware + def render_answer_file(self): + """ + Replace KVM_TEST_CDKEY (in the unattended file) with the cdkey + provided for this test and replace the KVM_TEST_MEDIUM with + the tree url or nfs address provided for this test. + + @return: Answer file contents + """ + error.base_context('Rendering final answer file') + error.context('Reading answer file %s' % self.unattended_file) + unattended_contents = open(self.unattended_file).read() + dummy_cdkey_re = r'\bKVM_TEST_CDKEY\b' + if re.search(dummy_cdkey_re, unattended_contents): + if self.cdkey: + unattended_contents = re.sub(dummy_cdkey_re, self.cdkey, + unattended_contents) + else: + print ("WARNING: 'cdkey' required but not specified for " + "this unattended installation") + + dummy_medium_re = r'\bKVM_TEST_MEDIUM\b' + if self.medium == "cdrom": + content = "cdrom" + elif self.medium == "url": + content = "url --url %s" % self.url + elif self.medium == "nfs": + content = "nfs --server=%s --dir=%s" % (self.nfs_server, + self.nfs_dir) + else: + raise ValueError("Unexpected installation medium %s" % self.url) + + unattended_contents = re.sub(dummy_medium_re, content, + unattended_contents) + + def replace_virtio_key(contents, dummy_re, attribute_name): + """ + Replace a virtio dummy string with contents. + + If install_virtio is not set, replace it with a dummy string. + + @param contents: Contents of the unattended file + @param dummy_re: Regular expression used to search on the. + unattended file contents. + @param env: Name of the environment variable. + """ + dummy_path = "C:" + driver = getattr(self, attribute_name, '') + + if re.search(dummy_re, contents): + if self.install_virtio == "yes": + if driver.endswith("msi"): + driver = 'msiexec /passive /package ' + driver + else: + try: + # Let's escape windows style paths properly + drive, path = driver.split(":") + driver = drive + ":" + re.escape(path) + except: + pass + contents = re.sub(dummy_re, driver, contents) + else: + contents = re.sub(dummy_re, dummy_path, contents) + return contents + + vdict = {r'\bKVM_TEST_STORAGE_DRIVER_PATH\b': + 'virtio_storage_path', + r'\bKVM_TEST_NETWORK_DRIVER_PATH\b': + 'virtio_network_path', + r'\bKVM_TEST_VIRTIO_NETWORK_INSTALLER\b': + 'virtio_network_installer_path'} + + for vkey in vdict: + unattended_contents = replace_virtio_key( + contents=unattended_contents, + dummy_re=vkey, + attribute_name=vdict[vkey]) + + logging.debug("Unattended install contents:") + for line in unattended_contents.splitlines(): + logging.debug(line) + return unattended_contents + + + def setup_boot_disk(self): + answer_contents = self.render_answer_file() + + if self.unattended_file.endswith('.sif'): + dest_fname = 'winnt.sif' + setup_file = 'winnt.bat' + boot_disk = FloppyDisk(self.floppy, self.qemu_img_binary, + self.tmpdir) + boot_disk.setup_answer_file(dest_fname, answer_contents) + setup_file_path = os.path.join(self.unattended_dir, setup_file) + boot_disk.copy_to(setup_file_path) + if self.install_virtio == "yes": + boot_disk.setup_virtio_win2003(self.virtio_floppy, + self.virtio_oemsetup_id) + boot_disk.copy_to(self.finish_program) + + elif self.unattended_file.endswith('.ks'): + # Red Hat kickstart install + dest_fname = 'ks.cfg' + if self.cdrom_unattended: + boot_disk = CdromDisk(self.cdrom_unattended, self.tmpdir) + elif self.floppy: + boot_disk = FloppyDisk(self.floppy, self.qemu_img_binary, + self.tmpdir) + else: + raise ValueError("Neither cdrom_unattended nor floppy set " + "on the config file, please verify") + boot_disk.setup_answer_file(dest_fname, answer_contents) + + elif self.unattended_file.endswith('.xml'): + if "autoyast" in self.extra_params: + # SUSE autoyast install + dest_fname = "autoinst.xml" + if self.cdrom_unattended: + boot_disk = CdromDisk(self.cdrom_unattended) + elif self.floppy: + boot_disk = FloppyDisk(self.floppy, self.qemu_img_binary, + self.tmpdir) + else: + raise ValueError("Neither cdrom_unattended nor floppy set " + "on the config file, please verify") + boot_disk.setup_answer_file(dest_fname, answer_contents) + + else: + # Windows unattended install + dest_fname = "autounattend.xml" + boot_disk = FloppyDisk(self.floppy, self.qemu_img_binary, + self.tmpdir) + boot_disk.setup_answer_file(dest_fname, answer_contents) + if self.install_virtio == "yes": + boot_disk.setup_virtio_win2008(self.virtio_floppy) + boot_disk.copy_to(self.finish_program) + + else: + raise ValueError('Unknown answer file type: %s' % + self.unattended_file) + + boot_disk.close() + + + @error.context_aware + def setup_cdrom(self): + """ + Mount cdrom and copy vmlinuz and initrd.img. + """ + error.context("Copying vmlinuz and initrd.img from install cdrom %s" % + self.cdrom_cd1) + m_cmd = ('mount -t iso9660 -v -o loop,ro %s %s' % + (self.cdrom_cd1, self.cdrom_cd1_mount)) + utils.run(m_cmd) + + try: + if not os.path.isdir(self.image_path): + os.makedirs(self.image_path) + kernel_fetch_cmd = ("cp %s/%s/%s %s" % + (self.cdrom_cd1_mount, self.boot_path, + os.path.basename(self.kernel), self.kernel)) + utils.run(kernel_fetch_cmd) + initrd_fetch_cmd = ("cp %s/%s/%s %s" % + (self.cdrom_cd1_mount, self.boot_path, + os.path.basename(self.initrd), self.initrd)) + utils.run(initrd_fetch_cmd) + finally: + cleanup(self.cdrom_cd1_mount) + + + @error.context_aware + def setup_url(self): + """ + Download the vmlinuz and initrd.img from URL. + """ + error.context("downloading vmlinuz and initrd.img from %s" % self.url) + os.chdir(self.image_path) + kernel_fetch_cmd = "wget -q %s/%s/%s" % (self.url, self.boot_path, + os.path.basename(self.kernel)) + initrd_fetch_cmd = "wget -q %s/%s/%s" % (self.url, self.boot_path, + os.path.basename(self.initrd)) + + if os.path.exists(self.kernel): + os.remove(self.kernel) + if os.path.exists(self.initrd): + os.remove(self.initrd) + + utils.run(kernel_fetch_cmd) + utils.run(initrd_fetch_cmd) + + + def setup_nfs(self): + """ + Copy the vmlinuz and initrd.img from nfs. + """ + error.context("copying the vmlinuz and initrd.img from NFS share") + + m_cmd = ("mount %s:%s %s -o ro" % + (self.nfs_server, self.nfs_dir, self.nfs_mount)) + utils.run(m_cmd) + + try: + kernel_fetch_cmd = ("cp %s/%s/%s %s" % + (self.nfs_mount, self.boot_path, + os.path.basename(self.kernel), self.image_path)) + utils.run(kernel_fetch_cmd) + initrd_fetch_cmd = ("cp %s/%s/%s %s" % + (self.nfs_mount, self.boot_path, + os.path.basename(self.initrd), self.image_path)) + utils.run(initrd_fetch_cmd) + finally: + cleanup(self.nfs_mount) + + + def setup(self): + """ + Configure the environment for unattended install. + + Uses an appropriate strategy according to each install model. + """ + logging.info("Starting unattended install setup") + display_attributes(self) + + if self.unattended_file and (self.floppy or self.cdrom_unattended): + self.setup_boot_disk() + if self.medium == "cdrom": + if self.kernel and self.initrd: + self.setup_cdrom() + elif self.medium == "url": + self.setup_url() + elif self.medium == "nfs": + self.setup_nfs() + else: + raise ValueError("Unexpected installation method %s" % + self.medium) + + +class HugePageConfig(object): + def __init__(self, params): + """ + Gets environment variable values and calculates the target number + of huge memory pages. + + @param params: Dict like object containing parameters for the test. + """ + self.vms = len(params.objects("vms")) + self.mem = int(params.get("mem")) + self.max_vms = int(params.get("max_vms", 0)) + self.hugepage_path = '/mnt/kvm_hugepage' + self.hugepage_size = self.get_hugepage_size() + self.target_hugepages = self.get_target_hugepages() + self.kernel_hp_file = '/proc/sys/vm/nr_hugepages' + + + def get_hugepage_size(self): + """ + Get the current system setting for huge memory page size. + """ + meminfo = open('/proc/meminfo', 'r').readlines() + huge_line_list = [h for h in meminfo if h.startswith("Hugepagesize")] + try: + return int(huge_line_list[0].split()[1]) + except ValueError, e: + raise ValueError("Could not get huge page size setting from " + "/proc/meminfo: %s" % e) + + + def get_target_hugepages(self): + """ + Calculate the target number of hugepages for testing purposes. + """ + if self.vms < self.max_vms: + self.vms = self.max_vms + # memory of all VMs plus qemu overhead of 64MB per guest + vmsm = (self.vms * self.mem) + (self.vms * 64) + return int(vmsm * 1024 / self.hugepage_size) + + + @error.context_aware + def set_hugepages(self): + """ + Sets the hugepage limit to the target hugepage value calculated. + """ + error.context("setting hugepages limit to %s" % self.target_hugepages) + hugepage_cfg = open(self.kernel_hp_file, "r+") + hp = hugepage_cfg.readline() + while int(hp) < self.target_hugepages: + loop_hp = hp + hugepage_cfg.write(str(self.target_hugepages)) + hugepage_cfg.flush() + hugepage_cfg.seek(0) + hp = int(hugepage_cfg.readline()) + if loop_hp == hp: + raise ValueError("Cannot set the kernel hugepage setting " + "to the target value of %d hugepages." % + self.target_hugepages) + hugepage_cfg.close() + logging.debug("Successfuly set %s large memory pages on host ", + self.target_hugepages) + + + @error.context_aware + def mount_hugepage_fs(self): + """ + Verify if there's a hugetlbfs mount set. If there's none, will set up + a hugetlbfs mount using the class attribute that defines the mount + point. + """ + error.context("mounting hugepages path") + if not os.path.ismount(self.hugepage_path): + if not os.path.isdir(self.hugepage_path): + os.makedirs(self.hugepage_path) + cmd = "mount -t hugetlbfs none %s" % self.hugepage_path + utils.system(cmd) + + + def setup(self): + logging.debug("Number of VMs this test will use: %d", self.vms) + logging.debug("Amount of memory used by each vm: %s", self.mem) + logging.debug("System setting for large memory page size: %s", + self.hugepage_size) + logging.debug("Number of large memory pages needed for this test: %s", + self.target_hugepages) + self.set_hugepages() + self.mount_hugepage_fs() + + + @error.context_aware + def cleanup(self): + error.context("trying to dealocate hugepage memory") + try: + utils.system("umount %s" % self.hugepage_path) + except error.CmdError: + return + utils.system("echo 0 > %s" % self.kernel_hp_file) + logging.debug("Hugepage memory successfuly dealocated") + + +class EnospcConfig(object): + """ + Performs setup for the test enospc. This is a borg class, similar to a + singleton. The idea is to keep state in memory for when we call cleanup() + on postprocessing. + """ + __shared_state = {} + def __init__(self, test, params): + self.__dict__ = self.__shared_state + root_dir = test.bindir + self.tmpdir = test.tmpdir + self.qemu_img_binary = params.get('qemu_img_binary') + if not os.path.isfile(self.qemu_img_binary): + self.qemu_img_binary = os.path.join(root_dir, + self.qemu_img_binary) + self.raw_file_path = os.path.join(self.tmpdir, 'enospc.raw') + # Here we're trying to choose fairly explanatory names so it's less + # likely that we run in conflict with other devices in the system + self.vgtest_name = params.get("vgtest_name") + self.lvtest_name = params.get("lvtest_name") + self.lvtest_device = "/dev/%s/%s" % (self.vgtest_name, self.lvtest_name) + image_dir = os.path.dirname(params.get("image_name")) + self.qcow_file_path = os.path.join(image_dir, 'enospc.qcow2') + try: + getattr(self, 'loopback') + except AttributeError: + self.loopback = '' + + + @error.context_aware + def setup(self): + logging.debug("Starting enospc setup") + error.context("performing enospc setup") + display_attributes(self) + # Double check if there aren't any leftovers + self.cleanup() + try: + utils.run("%s create -f raw %s 10G" % + (self.qemu_img_binary, self.raw_file_path)) + # Associate a loopback device with the raw file. + # Subject to race conditions, that's why try here to associate + # it with the raw file as quickly as possible + l_result = utils.run("losetup -f") + utils.run("losetup -f %s" % self.raw_file_path) + self.loopback = l_result.stdout.strip() + # Add the loopback device configured to the list of pvs + # recognized by LVM + utils.run("pvcreate %s" % self.loopback) + utils.run("vgcreate %s %s" % (self.vgtest_name, self.loopback)) + # Create an lv inside the vg with starting size of 200M + utils.run("lvcreate -L 200M -n %s %s" % + (self.lvtest_name, self.vgtest_name)) + # Create a 10GB qcow2 image in the logical volume + utils.run("%s create -f qcow2 %s 10G" % + (self.qemu_img_binary, self.lvtest_device)) + # Let's symlink the logical volume with the image name that autotest + # expects this device to have + os.symlink(self.lvtest_device, self.qcow_file_path) + except Exception, e: + self.cleanup() + raise + + @error.context_aware + def cleanup(self): + error.context("performing enospc cleanup") + if os.path.isfile(self.lvtest_device): + utils.run("fuser -k %s" % self.lvtest_device) + time.sleep(2) + l_result = utils.run("lvdisplay") + # Let's remove all volumes inside the volume group created + if self.lvtest_name in l_result.stdout: + utils.run("lvremove -f %s" % self.lvtest_device) + # Now, removing the volume group itself + v_result = utils.run("vgdisplay") + if self.vgtest_name in v_result.stdout: + utils.run("vgremove -f %s" % self.vgtest_name) + # Now, if we can, let's remove the physical volume from lvm list + if self.loopback: + p_result = utils.run("pvdisplay") + if self.loopback in p_result.stdout: + utils.run("pvremove -f %s" % self.loopback) + l_result = utils.run('losetup -a') + if self.loopback and (self.loopback in l_result.stdout): + try: + utils.run("losetup -d %s" % self.loopback) + except error.CmdError: + logging.error("Failed to liberate loopback %s", self.loopback) + if os.path.islink(self.qcow_file_path): + os.remove(self.qcow_file_path) + if os.path.isfile(self.raw_file_path): + os.remove(self.raw_file_path) diff --git a/client/virt/virt_test_utils.py b/client/virt/virt_test_utils.py new file mode 100644 index 0000000..f3d77ae --- /dev/null +++ b/client/virt/virt_test_utils.py @@ -0,0 +1,754 @@ +""" +High-level KVM test utility functions. + +This module is meant to reduce code size by performing common test procedures. +Generally, code here should look like test code. +More specifically: + - Functions in this module should raise exceptions if things go wrong + (unlike functions in kvm_utils.py and kvm_vm.py which report failure via + their returned values). + - Functions in this module may use logging.info(), in addition to + logging.debug() and logging.error(), to log messages the user may be + interested in (unlike kvm_utils.py and kvm_vm.py which use + logging.debug() for anything that isn't an error). + - Functions in this module typically use functions and classes from + lower-level modules (e.g. kvm_utils.py, kvm_vm.py, kvm_subprocess.py). + - Functions in this module should not be used by lower-level modules. + - Functions in this module should be used in the right context. + For example, a function should not be used where it may display + misleading or inaccurate info or debug messages. + +@copyright: 2008-2009 Red Hat Inc. +""" + +import time, os, logging, re, signal +from autotest_lib.client.bin import utils +from autotest_lib.client.common_lib import error +from autotest_lib.client.tools import scan_results +import virt_utils, virt_vm, aexpect + + +def get_living_vm(env, vm_name): + """ + Get a VM object from the environment and make sure it's alive. + + @param env: Dictionary with test environment. + @param vm_name: Name of the desired VM object. + @return: A VM object. + """ + vm = env.get_vm(vm_name) + if not vm: + raise error.TestError("VM '%s' not found in environment" % vm_name) + if not vm.is_alive(): + raise error.TestError("VM '%s' seems to be dead; test requires a " + "living VM" % vm_name) + return vm + + +def wait_for_login(vm, nic_index=0, timeout=240, start=0, step=2, serial=None): + """ + Try logging into a VM repeatedly. Stop on success or when timeout expires. + + @param vm: VM object. + @param nic_index: Index of NIC to access in the VM. + @param timeout: Time to wait before giving up. + @param serial: Whether to use a serial connection instead of a remote + (ssh, rss) one. + @return: A shell session object. + """ + end_time = time.time() + timeout + session = None + if serial: + type = 'serial' + logging.info("Trying to log into guest %s using serial connection," + " timeout %ds", vm.name, timeout) + time.sleep(start) + while time.time() < end_time: + try: + session = vm.serial_login() + break + except virt_utils.LoginError, e: + logging.debug(e) + time.sleep(step) + else: + type = 'remote' + logging.info("Trying to log into guest %s using remote connection," + " timeout %ds", vm.name, timeout) + time.sleep(start) + while time.time() < end_time: + try: + session = vm.login(nic_index=nic_index) + break + except (virt_utils.LoginError, virt_vm.VMError), e: + logging.debug(e) + time.sleep(step) + if not session: + raise error.TestFail("Could not log into guest %s using %s connection" % + (vm.name, type)) + logging.info("Logged into guest %s using %s connection", vm.name, type) + return session + + +def reboot(vm, session, method="shell", sleep_before_reset=10, nic_index=0, + timeout=240): + """ + Reboot the VM and wait for it to come back up by trying to log in until + timeout expires. + + @param vm: VM object. + @param session: A shell session object. + @param method: Reboot method. Can be "shell" (send a shell reboot + command) or "system_reset" (send a system_reset monitor command). + @param nic_index: Index of NIC to access in the VM, when logging in after + rebooting. + @param timeout: Time to wait before giving up (after rebooting). + @return: A new shell session object. + """ + if method == "shell": + # Send a reboot command to the guest's shell + session.sendline(vm.get_params().get("reboot_command")) + logging.info("Reboot command sent. Waiting for guest to go down...") + elif method == "system_reset": + # Sleep for a while before sending the command + time.sleep(sleep_before_reset) + # Clear the event list of all QMP monitors + monitors = [m for m in vm.monitors if m.protocol == "qmp"] + for m in monitors: + m.clear_events() + # Send a system_reset monitor command + vm.monitor.cmd("system_reset") + logging.info("Monitor command system_reset sent. Waiting for guest to " + "go down...") + # Look for RESET QMP events + time.sleep(1) + for m in monitors: + if not m.get_event("RESET"): + raise error.TestFail("RESET QMP event not received after " + "system_reset (monitor '%s')" % m.name) + else: + logging.info("RESET QMP event received") + else: + logging.error("Unknown reboot method: %s", method) + + # Wait for the session to become unresponsive and close it + if not virt_utils.wait_for(lambda: not session.is_responsive(timeout=30), + 120, 0, 1): + raise error.TestFail("Guest refuses to go down") + session.close() + + # Try logging into the guest until timeout expires + logging.info("Guest is down. Waiting for it to go up again, timeout %ds", + timeout) + session = vm.wait_for_login(nic_index, timeout=timeout) + logging.info("Guest is up again") + return session + + +def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp", + mig_cancel=False, offline=False, stable_check=False, + clean=False, save_path=None, dest_host='localhost', mig_port=None): + """ + Migrate a VM locally and re-register it in the environment. + + @param vm: The VM to migrate. + @param env: The environment dictionary. If omitted, the migrated VM will + not be registered. + @param mig_timeout: timeout value for migration. + @param mig_protocol: migration protocol + @param mig_cancel: Test migrate_cancel or not when protocol is tcp. + @param dest_host: Destination host (defaults to 'localhost'). + @param mig_port: Port that will be used for migration. + @return: The post-migration VM, in case of same host migration, True in + case of multi-host migration. + """ + def mig_finished(): + o = vm.monitor.info("migrate") + if isinstance(o, str): + return "status: active" not in o + else: + return o.get("status") != "active" + + def mig_succeeded(): + o = vm.monitor.info("migrate") + if isinstance(o, str): + return "status: completed" in o + else: + return o.get("status") == "completed" + + def mig_failed(): + o = vm.monitor.info("migrate") + if isinstance(o, str): + return "status: failed" in o + else: + return o.get("status") == "failed" + + def mig_cancelled(): + o = vm.monitor.info("migrate") + if isinstance(o, str): + return ("Migration status: cancelled" in o or + "Migration status: canceled" in o) + else: + return (o.get("status") == "cancelled" or + o.get("status") == "canceled") + + def wait_for_migration(): + if not virt_utils.wait_for(mig_finished, mig_timeout, 2, 2, + "Waiting for migration to finish..."): + raise error.TestFail("Timeout expired while waiting for migration " + "to finish") + + if dest_host == 'localhost': + dest_vm = vm.clone() + + if (dest_host == 'localhost') and stable_check: + # Pause the dest vm after creation + dest_vm.params['extra_params'] = (dest_vm.params.get('extra_params','') + + ' -S') + + if dest_host == 'localhost': + dest_vm.create(migration_mode=mig_protocol, mac_source=vm) + + try: + try: + if mig_protocol == "tcp": + if dest_host == 'localhost': + uri = "tcp:localhost:%d" % dest_vm.migration_port + else: + uri = 'tcp:%s:%d' % (dest_host, mig_port) + elif mig_protocol == "unix": + uri = "unix:%s" % dest_vm.migration_file + elif mig_protocol == "exec": + uri = '"exec:nc localhost %s"' % dest_vm.migration_port + + if offline: + vm.monitor.cmd("stop") + vm.monitor.migrate(uri) + + if mig_cancel: + time.sleep(2) + vm.monitor.cmd("migrate_cancel") + if not virt_utils.wait_for(mig_cancelled, 60, 2, 2, + "Waiting for migration " + "cancellation"): + raise error.TestFail("Failed to cancel migration") + if offline: + vm.monitor.cmd("cont") + if dest_host == 'localhost': + dest_vm.destroy(gracefully=False) + return vm + else: + wait_for_migration() + if (dest_host == 'localhost') and stable_check: + save_path = None or "/tmp" + save1 = os.path.join(save_path, "src") + save2 = os.path.join(save_path, "dst") + + vm.save_to_file(save1) + dest_vm.save_to_file(save2) + + # Fail if we see deltas + md5_save1 = utils.hash_file(save1) + md5_save2 = utils.hash_file(save2) + if md5_save1 != md5_save2: + raise error.TestFail("Mismatch of VM state before " + "and after migration") + + if (dest_host == 'localhost') and offline: + dest_vm.monitor.cmd("cont") + except: + if dest_host == 'localhost': + dest_vm.destroy() + raise + + finally: + if (dest_host == 'localhost') and stable_check and clean: + logging.debug("Cleaning the state files") + if os.path.isfile(save1): + os.remove(save1) + if os.path.isfile(save2): + os.remove(save2) + + # Report migration status + if mig_succeeded(): + logging.info("Migration finished successfully") + elif mig_failed(): + raise error.TestFail("Migration failed") + else: + raise error.TestFail("Migration ended with unknown status") + + if dest_host == 'localhost': + if "paused" in dest_vm.monitor.info("status"): + logging.debug("Destination VM is paused, resuming it...") + dest_vm.monitor.cmd("cont") + + # Kill the source VM + vm.destroy(gracefully=False) + + # Replace the source VM with the new cloned VM + if (dest_host == 'localhost') and (env is not None): + env.register_vm(vm.name, dest_vm) + + # Return the new cloned VM + if dest_host == 'localhost': + return dest_vm + else: + return vm + + +def stop_windows_service(session, service, timeout=120): + """ + Stop a Windows service using sc. + If the service is already stopped or is not installed, do nothing. + + @param service: The name of the service + @param timeout: Time duration to wait for service to stop + @raise error.TestError: Raised if the service can't be stopped + """ + end_time = time.time() + timeout + while time.time() < end_time: + o = session.cmd_output("sc stop %s" % service, timeout=60) + # FAILED 1060 means the service isn't installed. + # FAILED 1062 means the service hasn't been started. + if re.search(r"\bFAILED (1060|1062)\b", o, re.I): + break + time.sleep(1) + else: + raise error.TestError("Could not stop service '%s'" % service) + + +def start_windows_service(session, service, timeout=120): + """ + Start a Windows service using sc. + If the service is already running, do nothing. + If the service isn't installed, fail. + + @param service: The name of the service + @param timeout: Time duration to wait for service to start + @raise error.TestError: Raised if the service can't be started + """ + end_time = time.time() + timeout + while time.time() < end_time: + o = session.cmd_output("sc start %s" % service, timeout=60) + # FAILED 1060 means the service isn't installed. + if re.search(r"\bFAILED 1060\b", o, re.I): + raise error.TestError("Could not start service '%s' " + "(service not installed)" % service) + # FAILED 1056 means the service is already running. + if re.search(r"\bFAILED 1056\b", o, re.I): + break + time.sleep(1) + else: + raise error.TestError("Could not start service '%s'" % service) + + +def get_time(session, time_command, time_filter_re, time_format): + """ + Return the host time and guest time. If the guest time cannot be fetched + a TestError exception is raised. + + Note that the shell session should be ready to receive commands + (i.e. should "display" a command prompt and should be done with all + previous commands). + + @param session: A shell session. + @param time_command: Command to issue to get the current guest time. + @param time_filter_re: Regex filter to apply on the output of + time_command in order to get the current time. + @param time_format: Format string to pass to time.strptime() with the + result of the regex filter. + @return: A tuple containing the host time and guest time. + """ + if len(re.findall("ntpdate|w32tm", time_command)) == 0: + host_time = time.time() + s = session.cmd_output(time_command) + + try: + s = re.findall(time_filter_re, s)[0] + except IndexError: + logging.debug("The time string from guest is:\n%s", s) + raise error.TestError("The time string from guest is unexpected.") + except Exception, e: + logging.debug("(time_filter_re, time_string): (%s, %s)", + time_filter_re, s) + raise e + + guest_time = time.mktime(time.strptime(s, time_format)) + else: + o = session.cmd(time_command) + if re.match('ntpdate', time_command): + offset = re.findall('offset (.*) sec', o)[0] + host_main, host_mantissa = re.findall(time_filter_re, o)[0] + host_time = (time.mktime(time.strptime(host_main, time_format)) + + float("0.%s" % host_mantissa)) + guest_time = host_time + float(offset) + else: + guest_time = re.findall(time_filter_re, o)[0] + offset = re.findall("o:(.*)s", o)[0] + if re.match('PM', guest_time): + hour = re.findall('\d+ (\d+):', guest_time)[0] + hour = str(int(hour) + 12) + guest_time = re.sub('\d+\s\d+:', "\d+\s%s:" % hour, + guest_time)[:-3] + else: + guest_time = guest_time[:-3] + guest_time = time.mktime(time.strptime(guest_time, time_format)) + host_time = guest_time - float(offset) + + return (host_time, guest_time) + + +def get_memory_info(lvms): + """ + Get memory information from host and guests in format: + Host: memfree = XXXM; Guests memsh = {XXX,XXX,...} + + @params lvms: List of VM objects + @return: String with memory info report + """ + if not isinstance(lvms, list): + raise error.TestError("Invalid list passed to get_stat: %s " % lvms) + + try: + meminfo = "Host: memfree = " + meminfo += str(int(utils.freememtotal()) / 1024) + "M; " + meminfo += "swapfree = " + mf = int(utils.read_from_meminfo("SwapFree")) / 1024 + meminfo += str(mf) + "M; " + except Exception, e: + raise error.TestFail("Could not fetch host free memory info, " + "reason: %s" % e) + + meminfo += "Guests memsh = {" + for vm in lvms: + shm = vm.get_shared_meminfo() + if shm is None: + raise error.TestError("Could not get shared meminfo from " + "VM %s" % vm) + meminfo += "%dM; " % shm + meminfo = meminfo[0:-2] + "}" + + return meminfo + + +def run_autotest(vm, session, control_path, timeout, outputdir, params): + """ + Run an autotest control file inside a guest (linux only utility). + + @param vm: VM object. + @param session: A shell session on the VM provided. + @param control_path: A path to an autotest control file. + @param timeout: Timeout under which the autotest control file must complete. + @param outputdir: Path on host where we should copy the guest autotest + results to. + + The following params is used by the migration + @param params: Test params used in the migration test + """ + def copy_if_hash_differs(vm, local_path, remote_path): + """ + Copy a file to a guest if it doesn't exist or if its MD5sum differs. + + @param vm: VM object. + @param local_path: Local path. + @param remote_path: Remote path. + """ + local_hash = utils.hash_file(local_path) + basename = os.path.basename(local_path) + output = session.cmd_output("md5sum %s" % remote_path) + if "such file" in output: + remote_hash = "0" + elif output: + remote_hash = output.split()[0] + else: + logging.warning("MD5 check for remote path %s did not return.", + remote_path) + # Let's be a little more lenient here and see if it wasn't a + # temporary problem + remote_hash = "0" + if remote_hash != local_hash: + logging.debug("Copying %s to guest", basename) + vm.copy_files_to(local_path, remote_path) + + + def extract(vm, remote_path, dest_dir="."): + """ + Extract a .tar.bz2 file on the guest. + + @param vm: VM object + @param remote_path: Remote file path + @param dest_dir: Destination dir for the contents + """ + basename = os.path.basename(remote_path) + logging.info("Extracting %s...", basename) + e_cmd = "tar xjvf %s -C %s" % (remote_path, dest_dir) + session.cmd(e_cmd, timeout=120) + + + def get_results(): + """ + Copy autotest results present on the guest back to the host. + """ + logging.info("Trying to copy autotest results from guest") + guest_results_dir = os.path.join(outputdir, "guest_autotest_results") + if not os.path.exists(guest_results_dir): + os.mkdir(guest_results_dir) + vm.copy_files_from("%s/results/default/*" % autotest_path, + guest_results_dir) + + + def get_results_summary(): + """ + Get the status of the tests that were executed on the host and close + the session where autotest was being executed. + """ + output = session.cmd_output("cat results/*/status") + try: + results = scan_results.parse_results(output) + # Report test results + logging.info("Results (test, status, duration, info):") + for result in results: + logging.info(str(result)) + session.close() + return results + except Exception, e: + logging.error("Error processing guest autotest results: %s", e) + return None + + + if not os.path.isfile(control_path): + raise error.TestError("Invalid path to autotest control file: %s" % + control_path) + + migrate_background = params.get("migrate_background") == "yes" + if migrate_background: + mig_timeout = float(params.get("mig_timeout", "3600")) + mig_protocol = params.get("migration_protocol", "tcp") + + compressed_autotest_path = "/tmp/autotest.tar.bz2" + + # To avoid problems, let's make the test use the current AUTODIR + # (autotest client path) location + autotest_path = os.environ['AUTODIR'] + + # tar the contents of bindir/autotest + cmd = "tar cvjf %s %s/*" % (compressed_autotest_path, autotest_path) + # Until we have nested virtualization, we don't need the kvm test :) + cmd += " --exclude=%s/tests/kvm" % autotest_path + cmd += " --exclude=%s/results" % autotest_path + cmd += " --exclude=%s/tmp" % autotest_path + cmd += " --exclude=%s/control*" % autotest_path + cmd += " --exclude=*.pyc" + cmd += " --exclude=*.svn" + cmd += " --exclude=*.git" + utils.run(cmd) + + # Copy autotest.tar.bz2 + copy_if_hash_differs(vm, compressed_autotest_path, compressed_autotest_path) + + # Extract autotest.tar.bz2 + extract(vm, compressed_autotest_path, "/") + + vm.copy_files_to(control_path, os.path.join(autotest_path, 'control')) + + # Run the test + logging.info("Running autotest control file %s on guest, timeout %ss", + os.path.basename(control_path), timeout) + session.cmd("cd %s" % autotest_path) + try: + session.cmd("rm -f control.state") + session.cmd("rm -rf results/*") + except aexpect.ShellError: + pass + try: + bg = None + try: + logging.info("---------------- Test output ----------------") + if migrate_background: + mig_timeout = float(params.get("mig_timeout", "3600")) + mig_protocol = params.get("migration_protocol", "tcp") + + bg = virt_utils.Thread(session.cmd_output, + kwargs={'cmd': "bin/autotest control", + 'timeout': timeout, + 'print_func': logging.info}) + + bg.start() + + while bg.is_alive(): + logging.info("Tests is not ended, start a round of" + "migration ...") + vm.migrate(timeout=mig_timeout, protocol=mig_protocol) + else: + session.cmd_output("bin/autotest control", timeout=timeout, + print_func=logging.info) + finally: + logging.info("------------- End of test output ------------") + if migrate_background and bg: + bg.join() + except aexpect.ShellTimeoutError: + if vm.is_alive(): + get_results() + get_results_summary() + raise error.TestError("Timeout elapsed while waiting for job to " + "complete") + else: + raise error.TestError("Autotest job on guest failed " + "(VM terminated during job)") + except aexpect.ShellProcessTerminatedError: + get_results() + raise error.TestError("Autotest job on guest failed " + "(Remote session terminated during job)") + + results = get_results_summary() + get_results() + + # Make a list of FAIL/ERROR/ABORT results (make sure FAIL results appear + # before ERROR results, and ERROR results appear before ABORT results) + bad_results = [r[0] for r in results if r[1] == "FAIL"] + bad_results += [r[0] for r in results if r[1] == "ERROR"] + bad_results += [r[0] for r in results if r[1] == "ABORT"] + + # Fail the test if necessary + if not results: + raise error.TestFail("Autotest control file run did not produce any " + "recognizable results") + if bad_results: + if len(bad_results) == 1: + e_msg = ("Test %s failed during control file execution" % + bad_results[0]) + else: + e_msg = ("Tests %s failed during control file execution" % + " ".join(bad_results)) + raise error.TestFail(e_msg) + + +def get_loss_ratio(output): + """ + Get the packet loss ratio from the output of ping +. + @param output: Ping output. + """ + try: + return int(re.findall('(\d+)% packet loss', output)[0]) + except IndexError: + logging.debug(output) + return -1 + + +def raw_ping(command, timeout, session, output_func): + """ + Low-level ping command execution. + + @param command: Ping command. + @param timeout: Timeout of the ping command. + @param session: Local executon hint or session to execute the ping command. + """ + if session is None: + process = aexpect.run_bg(command, output_func=output_func, + timeout=timeout) + + # Send SIGINT signal to notify the timeout of running ping process, + # Because ping have the ability to catch the SIGINT signal so we can + # always get the packet loss ratio even if timeout. + if process.is_alive(): + virt_utils.kill_process_tree(process.get_pid(), signal.SIGINT) + + status = process.get_status() + output = process.get_output() + + process.close() + return status, output + else: + output = "" + try: + output = session.cmd_output(command, timeout=timeout, + print_func=output_func) + except aexpect.ShellTimeoutError: + # Send ctrl+c (SIGINT) through ssh session + session.send("\003") + try: + output2 = session.read_up_to_prompt(print_func=output_func) + output += output2 + except aexpect.ExpectTimeoutError, e: + output += e.output + # We also need to use this session to query the return value + session.send("\003") + + session.sendline(session.status_test_command) + try: + o2 = session.read_up_to_prompt() + except aexpect.ExpectError: + status = -1 + else: + try: + status = int(re.findall("\d+", o2)[0]) + except: + status = -1 + + return status, output + + +def ping(dest=None, count=None, interval=None, interface=None, + packetsize=None, ttl=None, hint=None, adaptive=False, + broadcast=False, flood=False, timeout=0, + output_func=logging.debug, session=None): + """ + Wrapper of ping. + + @param dest: Destination address. + @param count: Count of icmp packet. + @param interval: Interval of two icmp echo request. + @param interface: Specified interface of the source address. + @param packetsize: Packet size of icmp. + @param ttl: IP time to live. + @param hint: Path mtu discovery hint. + @param adaptive: Adaptive ping flag. + @param broadcast: Broadcast ping flag. + @param flood: Flood ping flag. + @param timeout: Timeout for the ping command. + @param output_func: Function used to log the result of ping. + @param session: Local executon hint or session to execute the ping command. + """ + if dest is not None: + command = "ping %s " % dest + else: + command = "ping localhost " + if count is not None: + command += " -c %s" % count + if interval is not None: + command += " -i %s" % interval + if interface is not None: + command += " -I %s" % interface + if packetsize is not None: + command += " -s %s" % packetsize + if ttl is not None: + command += " -t %s" % ttl + if hint is not None: + command += " -M %s" % hint + if adaptive: + command += " -A" + if broadcast: + command += " -b" + if flood: + command += " -f -q" + output_func = None + + return raw_ping(command, timeout, session, output_func) + + +def get_linux_ifname(session, mac_address): + """ + Get the interface name through the mac address. + + @param session: session to the virtual machine + @mac_address: the macaddress of nic + """ + + output = session.cmd_output("ifconfig -a") + + try: + ethname = re.findall("(\w+)\s+Link.*%s" % mac_address, output, + re.IGNORECASE)[0] + return ethname + except: + return None diff --git a/client/virt/virt_utils.py b/client/virt/virt_utils.py new file mode 100644 index 0000000..26438df --- /dev/null +++ b/client/virt/virt_utils.py @@ -0,0 +1,1760 @@ +""" +KVM test utility functions. + +@copyright: 2008-2009 Red Hat Inc. +""" + +import time, string, random, socket, os, signal, re, logging, commands, cPickle +import fcntl, shelve, ConfigParser, threading, sys, UserDict +from autotest_lib.client.bin import utils, os_dep +from autotest_lib.client.common_lib import error, logging_config +import rss_client, virt_utils, aexpect + +try: + import koji + KOJI_INSTALLED = True +except ImportError: + KOJI_INSTALLED = False + + +def _lock_file(filename): + f = open(filename, "w") + fcntl.lockf(f, fcntl.LOCK_EX) + return f + + +def _unlock_file(f): + fcntl.lockf(f, fcntl.LOCK_UN) + f.close() + + +def is_vm(obj): + """ + Tests whether a given object is a VM object. + + @param obj: Python object. + """ + return obj.__class__.__name__ == "VM" + + +class Env(UserDict.IterableUserDict): + """ + A dict-like object containing global objects used by tests. + """ + def __init__(self, filename=None, version=0): + """ + Create an empty Env object or load an existing one from a file. + + If the version recorded in the file is lower than version, or if some + error occurs during unpickling, or if filename is not supplied, + create an empty Env object. + + @param filename: Path to an env file. + @param version: Required env version (int). + """ + UserDict.IterableUserDict.__init__(self) + empty = {"version": version} + if filename: + self._filename = filename + try: + f = open(filename, "r") + env = cPickle.load(f) + f.close() + if env.get("version", 0) >= version: + self.data = env + else: + logging.warn("Incompatible env file found. Not using it.") + self.data = empty + # Almost any exception can be raised during unpickling, so let's + # catch them all + except Exception, e: + logging.warn(e) + self.data = empty + else: + self.data = empty + + + def save(self, filename=None): + """ + Pickle the contents of the Env object into a file. + + @param filename: Filename to pickle the dict into. If not supplied, + use the filename from which the dict was loaded. + """ + filename = filename or self._filename + f = open(filename, "w") + cPickle.dump(self.data, f) + f.close() + + + def get_all_vms(self): + """ + Return a list of all VM objects in this Env object. + """ + return [o for o in self.values() if is_vm(o)] + + + def get_vm(self, name): + """ + Return a VM object by its name. + + @param name: VM name. + """ + return self.get("vm__%s" % name) + + + def register_vm(self, name, vm): + """ + Register a VM in this Env object. + + @param name: VM name. + @param vm: VM object. + """ + self["vm__%s" % name] = vm + + + def unregister_vm(self, name): + """ + Remove a given VM. + + @param name: VM name. + """ + del self["vm__%s" % name] + + + def register_installer(self, installer): + """ + Register a installer that was just run + + The installer will be available for other tests, so that + information about the installed KVM modules and qemu-kvm can be used by + them. + """ + self['last_installer'] = installer + + + def previous_installer(self): + """ + Return the last installer that was registered + """ + return self.get('last_installer') + + +class Params(UserDict.IterableUserDict): + """ + A dict-like object passed to every test. + """ + def objects(self, key): + """ + Return the names of objects defined using a given key. + + @param key: The name of the key whose value lists the objects + (e.g. 'nics'). + """ + return self.get(key, "").split() + + + def object_params(self, obj_name): + """ + Return a dict-like object containing the parameters of an individual + object. + + This method behaves as follows: the suffix '_' + obj_name is removed + from all key names that have it. Other key names are left unchanged. + The values of keys with the suffix overwrite the values of their + suffixless versions. + + @param obj_name: The name of the object (objects are listed by the + objects() method). + """ + suffix = "_" + obj_name + new_dict = self.copy() + for key in self: + if key.endswith(suffix): + new_key = key.split(suffix)[0] + new_dict[new_key] = self[key] + return new_dict + + +# Functions related to MAC/IP addresses + +def _open_mac_pool(lock_mode): + lock_file = open("/tmp/mac_lock", "w+") + fcntl.lockf(lock_file, lock_mode) + pool = shelve.open("/tmp/address_pool") + return pool, lock_file + + +def _close_mac_pool(pool, lock_file): + pool.close() + fcntl.lockf(lock_file, fcntl.LOCK_UN) + lock_file.close() + + +def _generate_mac_address_prefix(mac_pool): + """ + Generate a random MAC address prefix and add it to the MAC pool dictionary. + If there's a MAC prefix there already, do not update the MAC pool and just + return what's in there. By convention we will set KVM autotest MAC + addresses to start with 0x9a. + + @param mac_pool: The MAC address pool object. + @return: The MAC address prefix. + """ + if "prefix" in mac_pool: + prefix = mac_pool["prefix"] + logging.debug("Used previously generated MAC address prefix for this " + "host: %s", prefix) + else: + r = random.SystemRandom() + prefix = "9a:%02x:%02x:%02x:" % (r.randint(0x00, 0xff), + r.randint(0x00, 0xff), + r.randint(0x00, 0xff)) + mac_pool["prefix"] = prefix + logging.debug("Generated MAC address prefix for this host: %s", prefix) + return prefix + + +def generate_mac_address(vm_instance, nic_index): + """ + Randomly generate a MAC address and add it to the MAC address pool. + + Try to generate a MAC address based on a randomly generated MAC address + prefix and add it to a persistent dictionary. + key = VM instance + NIC index, value = MAC address + e.g. {'20100310-165222-Wt7l:0': '9a:5d:94:6a:9b:f9'} + + @param vm_instance: The instance attribute of a VM. + @param nic_index: The index of the NIC. + @return: MAC address string. + """ + mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX) + key = "%s:%s" % (vm_instance, nic_index) + if key in mac_pool: + mac = mac_pool[key] + else: + prefix = _generate_mac_address_prefix(mac_pool) + r = random.SystemRandom() + while key not in mac_pool: + mac = prefix + "%02x:%02x" % (r.randint(0x00, 0xff), + r.randint(0x00, 0xff)) + if mac in mac_pool.values(): + continue + mac_pool[key] = mac + logging.debug("Generated MAC address for NIC %s: %s", key, mac) + _close_mac_pool(mac_pool, lock_file) + return mac + + +def free_mac_address(vm_instance, nic_index): + """ + Remove a MAC address from the address pool. + + @param vm_instance: The instance attribute of a VM. + @param nic_index: The index of the NIC. + """ + mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX) + key = "%s:%s" % (vm_instance, nic_index) + if key in mac_pool: + logging.debug("Freeing MAC address for NIC %s: %s", key, mac_pool[key]) + del mac_pool[key] + _close_mac_pool(mac_pool, lock_file) + + +def set_mac_address(vm_instance, nic_index, mac): + """ + Set a MAC address in the pool. + + @param vm_instance: The instance attribute of a VM. + @param nic_index: The index of the NIC. + """ + mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX) + mac_pool["%s:%s" % (vm_instance, nic_index)] = mac + _close_mac_pool(mac_pool, lock_file) + + +def get_mac_address(vm_instance, nic_index): + """ + Return a MAC address from the pool. + + @param vm_instance: The instance attribute of a VM. + @param nic_index: The index of the NIC. + @return: MAC address string. + """ + mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_SH) + mac = mac_pool.get("%s:%s" % (vm_instance, nic_index)) + _close_mac_pool(mac_pool, lock_file) + return mac + + +def verify_ip_address_ownership(ip, macs, timeout=10.0): + """ + Use arping and the ARP cache to make sure a given IP address belongs to one + of the given MAC addresses. + + @param ip: An IP address. + @param macs: A list or tuple of MAC addresses. + @return: True iff ip is assigned to a MAC address in macs. + """ + # Compile a regex that matches the given IP address and any of the given + # MAC addresses + mac_regex = "|".join("(%s)" % mac for mac in macs) + regex = re.compile(r"\b%s\b.*\b(%s)\b" % (ip, mac_regex), re.IGNORECASE) + + # Check the ARP cache + o = commands.getoutput("%s -n" % find_command("arp")) + if regex.search(o): + return True + + # Get the name of the bridge device for arping + o = commands.getoutput("%s route get %s" % (find_command("ip"), ip)) + dev = re.findall("dev\s+\S+", o, re.IGNORECASE) + if not dev: + return False + dev = dev[0].split()[-1] + + # Send an ARP request + o = commands.getoutput("%s -f -c 3 -I %s %s" % + (find_command("arping"), dev, ip)) + return bool(regex.search(o)) + + +# Utility functions for dealing with external processes + +def find_command(cmd): + for dir in ["/usr/local/sbin", "/usr/local/bin", + "/usr/sbin", "/usr/bin", "/sbin", "/bin"]: + file = os.path.join(dir, cmd) + if os.path.exists(file): + return file + raise ValueError('Missing command: %s' % cmd) + + +def pid_exists(pid): + """ + Return True if a given PID exists. + + @param pid: Process ID number. + """ + try: + os.kill(pid, 0) + return True + except: + return False + + +def safe_kill(pid, signal): + """ + Attempt to send a signal to a given process that may or may not exist. + + @param signal: Signal number. + """ + try: + os.kill(pid, signal) + return True + except: + return False + + +def kill_process_tree(pid, sig=signal.SIGKILL): + """Signal a process and all of its children. + + If the process does not exist -- return. + + @param pid: The pid of the process to signal. + @param sig: The signal to send to the processes. + """ + if not safe_kill(pid, signal.SIGSTOP): + return + children = commands.getoutput("ps --ppid=%d -o pid=" % pid).split() + for child in children: + kill_process_tree(int(child), sig) + safe_kill(pid, sig) + safe_kill(pid, signal.SIGCONT) + + +def get_latest_kvm_release_tag(release_listing): + """ + Fetches the latest release tag for KVM. + + @param release_listing: URL that contains a list of the Source Forge + KVM project files. + """ + try: + release_page = utils.urlopen(release_listing) + data = release_page.read() + release_page.close() + rx = re.compile("kvm-(\d+).tar.gz", re.IGNORECASE) + matches = rx.findall(data) + # In all regexp matches to something that looks like a release tag, + # get the largest integer. That will be our latest release tag. + latest_tag = max(int(x) for x in matches) + return str(latest_tag) + except Exception, e: + message = "Could not fetch latest KVM release tag: %s" % str(e) + logging.error(message) + raise error.TestError(message) + + +def get_git_branch(repository, branch, srcdir, commit=None, lbranch=None): + """ + Retrieves a given git code repository. + + @param repository: Git repository URL + """ + logging.info("Fetching git [REP '%s' BRANCH '%s' COMMIT '%s'] -> %s", + repository, branch, commit, srcdir) + if not os.path.exists(srcdir): + os.makedirs(srcdir) + os.chdir(srcdir) + + if os.path.exists(".git"): + utils.system("git reset --hard") + else: + utils.system("git init") + + if not lbranch: + lbranch = branch + + utils.system("git fetch -q -f -u -t %s %s:%s" % + (repository, branch, lbranch)) + utils.system("git checkout %s" % lbranch) + if commit: + utils.system("git checkout %s" % commit) + + h = utils.system_output('git log --pretty=format:"%H" -1') + try: + desc = "tag %s" % utils.system_output("git describe") + except error.CmdError: + desc = "no tag found" + + logging.info("Commit hash for %s is %s (%s)", repository, h.strip(), desc) + return srcdir + + +def check_kvm_source_dir(source_dir): + """ + Inspects the kvm source directory and verifies its disposition. In some + occasions build may be dependant on the source directory disposition. + The reason why the return codes are numbers is that we might have more + changes on the source directory layout, so it's not scalable to just use + strings like 'old_repo', 'new_repo' and such. + + @param source_dir: Source code path that will be inspected. + """ + os.chdir(source_dir) + has_qemu_dir = os.path.isdir('qemu') + has_kvm_dir = os.path.isdir('kvm') + if has_qemu_dir: + logging.debug("qemu directory detected, source dir layout 1") + return 1 + if has_kvm_dir and not has_qemu_dir: + logging.debug("kvm directory detected, source dir layout 2") + return 2 + else: + raise error.TestError("Unknown source dir layout, cannot proceed.") + + +def get_virt_info(params, test): + vm_type = params.get('vm_type') + if vm_type == 'kvm': + # Get the KVM kernel module version and write it as a keyval + logging.debug("Fetching KVM module version...") + if os.path.exists("/dev/kvm"): + try: + kvm_version = open("/sys/module/kvm/version").read().strip() + except: + kvm_version = os.uname()[2] + else: + kvm_version = "Unknown" + logging.debug("KVM module not loaded") + logging.debug("KVM version: %s" % kvm_version) + test.write_test_keyval({"kvm_version": kvm_version}) + + # Get the KVM userspace version and write it as a keyval + logging.debug("Fetching KVM userspace version...") + qemu_path = virt_utils.get_path(test.bindir, params.get("qemu_binary", + "qemu")) + version_line = commands.getoutput("%s -help | head -n 1" % qemu_path) + matches = re.findall("[Vv]ersion .*?,", version_line) + if matches: + kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",") + else: + kvm_userspace_version = "Unknown" + logging.debug("Could not fetch KVM userspace version") + logging.debug("KVM userspace version: %s" % kvm_userspace_version) + test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version}) + + +# Functions and classes used for logging into guests and transferring files + +class LoginError(Exception): + def __init__(self, msg, output): + Exception.__init__(self, msg, output) + self.msg = msg + self.output = output + + def __str__(self): + return "%s (output: %r)" % (self.msg, self.output) + + +class LoginAuthenticationError(LoginError): + pass + + +class LoginTimeoutError(LoginError): + def __init__(self, output): + LoginError.__init__(self, "Login timeout expired", output) + + +class LoginProcessTerminatedError(LoginError): + def __init__(self, status, output): + LoginError.__init__(self, None, output) + self.status = status + + def __str__(self): + return ("Client process terminated (status: %s, output: %r)" % + (self.status, self.output)) + + +class LoginBadClientError(LoginError): + def __init__(self, client): + LoginError.__init__(self, None, None) + self.client = client + + def __str__(self): + return "Unknown remote shell client: %r" % self.client + + +class SCPError(Exception): + def __init__(self, msg, output): + Exception.__init__(self, msg, output) + self.msg = msg + self.output = output + + def __str__(self): + return "%s (output: %r)" % (self.msg, self.output) + + +class SCPAuthenticationError(SCPError): + pass + + +class SCPAuthenticationTimeoutError(SCPAuthenticationError): + def __init__(self, output): + SCPAuthenticationError.__init__(self, "Authentication timeout expired", + output) + + +class SCPTransferTimeoutError(SCPError): + def __init__(self, output): + SCPError.__init__(self, "Transfer timeout expired", output) + + +class SCPTransferFailedError(SCPError): + def __init__(self, status, output): + SCPError.__init__(self, None, output) + self.status = status + + def __str__(self): + return ("SCP transfer failed (status: %s, output: %r)" % + (self.status, self.output)) + + +def _remote_login(session, username, password, prompt, timeout=10): + """ + Log into a remote host (guest) using SSH or Telnet. Wait for questions + and provide answers. If timeout expires while waiting for output from the + child (e.g. a password prompt or a shell prompt) -- fail. + + @brief: Log into a remote host (guest) using SSH or Telnet. + + @param session: An Expect or ShellSession instance to operate on + @param username: The username to send in reply to a login prompt + @param password: The password to send in reply to a password prompt + @param prompt: The shell prompt that indicates a successful login + @param timeout: The maximal time duration (in seconds) to wait for each + step of the login procedure (i.e. the "Are you sure" prompt, the + password prompt, the shell prompt, etc) + @raise LoginTimeoutError: If timeout expires + @raise LoginAuthenticationError: If authentication fails + @raise LoginProcessTerminatedError: If the client terminates during login + @raise LoginError: If some other error occurs + """ + password_prompt_count = 0 + login_prompt_count = 0 + + while True: + try: + match, text = session.read_until_last_line_matches( + [r"[Aa]re you sure", r"[Pp]assword:\s*$", r"[Ll]ogin:\s*$", + r"[Cc]onnection.*closed", r"[Cc]onnection.*refused", + r"[Pp]lease wait", prompt], + timeout=timeout, internal_timeout=0.5) + if match == 0: # "Are you sure you want to continue connecting" + logging.debug("Got 'Are you sure...'; sending 'yes'") + session.sendline("yes") + continue + elif match == 1: # "password:" + if password_prompt_count == 0: + logging.debug("Got password prompt; sending '%s'", password) + session.sendline(password) + password_prompt_count += 1 + continue + else: + raise LoginAuthenticationError("Got password prompt twice", + text) + elif match == 2: # "login:" + if login_prompt_count == 0 and password_prompt_count == 0: + logging.debug("Got username prompt; sending '%s'", username) + session.sendline(username) + login_prompt_count += 1 + continue + else: + if login_prompt_count > 0: + msg = "Got username prompt twice" + else: + msg = "Got username prompt after password prompt" + raise LoginAuthenticationError(msg, text) + elif match == 3: # "Connection closed" + raise LoginError("Client said 'connection closed'", text) + elif match == 4: # "Connection refused" + raise LoginError("Client said 'connection refused'", text) + elif match == 5: # "Please wait" + logging.debug("Got 'Please wait'") + timeout = 30 + continue + elif match == 6: # prompt + logging.debug("Got shell prompt -- logged in") + break + except aexpect.ExpectTimeoutError, e: + raise LoginTimeoutError(e.output) + except aexpect.ExpectProcessTerminatedError, e: + raise LoginProcessTerminatedError(e.status, e.output) + + +def remote_login(client, host, port, username, password, prompt, linesep="\n", + log_filename=None, timeout=10): + """ + Log into a remote host (guest) using SSH/Telnet/Netcat. + + @param client: The client to use ('ssh', 'telnet' or 'nc') + @param host: Hostname or IP address + @param port: Port to connect to + @param username: Username (if required) + @param password: Password (if required) + @param prompt: Shell prompt (regular expression) + @param linesep: The line separator to use when sending lines + (e.g. '\\n' or '\\r\\n') + @param log_filename: If specified, log all output to this file + @param timeout: The maximal time duration (in seconds) to wait for + each step of the login procedure (i.e. the "Are you sure" prompt + or the password prompt) + @raise LoginBadClientError: If an unknown client is requested + @raise: Whatever _remote_login() raises + @return: A ShellSession object. + """ + if client == "ssh": + cmd = ("ssh -o UserKnownHostsFile=/dev/null " + "-o PreferredAuthentications=password -p %s %s@%s" % + (port, username, host)) + elif client == "telnet": + cmd = "telnet -l %s %s %s" % (username, host, port) + elif client == "nc": + cmd = "nc %s %s" % (host, port) + else: + raise LoginBadClientError(client) + + logging.debug("Trying to login with command '%s'", cmd) + session = aexpect.ShellSession(cmd, linesep=linesep, prompt=prompt) + try: + _remote_login(session, username, password, prompt, timeout) + except: + session.close() + raise + if log_filename: + session.set_output_func(log_line) + session.set_output_params((log_filename,)) + return session + + +def wait_for_login(client, host, port, username, password, prompt, linesep="\n", + log_filename=None, timeout=240, internal_timeout=10): + """ + Make multiple attempts to log into a remote host (guest) until one succeeds + or timeout expires. + + @param timeout: Total time duration to wait for a successful login + @param internal_timeout: The maximal time duration (in seconds) to wait for + each step of the login procedure (e.g. the "Are you sure" prompt + or the password prompt) + @see: remote_login() + @raise: Whatever remote_login() raises + @return: A ShellSession object. + """ + logging.debug("Attempting to log into %s:%s using %s (timeout %ds)", + host, port, client, timeout) + end_time = time.time() + timeout + while time.time() < end_time: + try: + return remote_login(client, host, port, username, password, prompt, + linesep, log_filename, internal_timeout) + except LoginError, e: + logging.debug(e) + time.sleep(2) + # Timeout expired; try one more time but don't catch exceptions + return remote_login(client, host, port, username, password, prompt, + linesep, log_filename, internal_timeout) + + +def _remote_scp(session, password, transfer_timeout=600, login_timeout=10): + """ + Transfer file(s) to a remote host (guest) using SCP. Wait for questions + and provide answers. If login_timeout expires while waiting for output + from the child (e.g. a password prompt), fail. If transfer_timeout expires + while waiting for the transfer to complete, fail. + + @brief: Transfer files using SCP, given a command line. + + @param session: An Expect or ShellSession instance to operate on + @param password: The password to send in reply to a password prompt. + @param transfer_timeout: The time duration (in seconds) to wait for the + transfer to complete. + @param login_timeout: The maximal time duration (in seconds) to wait for + each step of the login procedure (i.e. the "Are you sure" prompt or + the password prompt) + @raise SCPAuthenticationError: If authentication fails + @raise SCPTransferTimeoutError: If the transfer fails to complete in time + @raise SCPTransferFailedError: If the process terminates with a nonzero + exit code + @raise SCPError: If some other error occurs + """ + password_prompt_count = 0 + timeout = login_timeout + authentication_done = False + + while True: + try: + match, text = session.read_until_last_line_matches( + [r"[Aa]re you sure", r"[Pp]assword:\s*$", r"lost connection"], + timeout=timeout, internal_timeout=0.5) + if match == 0: # "Are you sure you want to continue connecting" + logging.debug("Got 'Are you sure...'; sending 'yes'") + session.sendline("yes") + continue + elif match == 1: # "password:" + if password_prompt_count == 0: + logging.debug("Got password prompt; sending '%s'", password) + session.sendline(password) + password_prompt_count += 1 + timeout = transfer_timeout + authentication_done = True + continue + else: + raise SCPAuthenticationError("Got password prompt twice", + text) + elif match == 2: # "lost connection" + raise SCPError("SCP client said 'lost connection'", text) + except aexpect.ExpectTimeoutError, e: + if authentication_done: + raise SCPTransferTimeoutError(e.output) + else: + raise SCPAuthenticationTimeoutError(e.output) + except aexpect.ExpectProcessTerminatedError, e: + if e.status == 0: + logging.debug("SCP process terminated with status 0") + break + else: + raise SCPTransferFailedError(e.status, e.output) + + +def remote_scp(command, password, log_filename=None, transfer_timeout=600, + login_timeout=10): + """ + Transfer file(s) to a remote host (guest) using SCP. + + @brief: Transfer files using SCP, given a command line. + + @param command: The command to execute + (e.g. "scp -r foobar root@localhost:/tmp/"). + @param password: The password to send in reply to a password prompt. + @param log_filename: If specified, log all output to this file + @param transfer_timeout: The time duration (in seconds) to wait for the + transfer to complete. + @param login_timeout: The maximal time duration (in seconds) to wait for + each step of the login procedure (i.e. the "Are you sure" prompt + or the password prompt) + @raise: Whatever _remote_scp() raises + """ + logging.debug("Trying to SCP with command '%s', timeout %ss", + command, transfer_timeout) + if log_filename: + output_func = log_line + output_params = (log_filename,) + else: + output_func = None + output_params = () + session = aexpect.Expect(command, + output_func=output_func, + output_params=output_params) + try: + _remote_scp(session, password, transfer_timeout, login_timeout) + finally: + session.close() + + +def scp_to_remote(host, port, username, password, local_path, remote_path, + log_filename=None, timeout=600): + """ + Copy files to a remote host (guest) through scp. + + @param host: Hostname or IP address + @param username: Username (if required) + @param password: Password (if required) + @param local_path: Path on the local machine where we are copying from + @param remote_path: Path on the remote machine where we are copying to + @param log_filename: If specified, log all output to this file + @param timeout: The time duration (in seconds) to wait for the transfer + to complete. + @raise: Whatever remote_scp() raises + """ + command = ("scp -v -o UserKnownHostsFile=/dev/null " + "-o PreferredAuthentications=password -r -P %s %s %s@%s:%s" % + (port, local_path, username, host, remote_path)) + remote_scp(command, password, log_filename, timeout) + + +def scp_from_remote(host, port, username, password, remote_path, local_path, + log_filename=None, timeout=600): + """ + Copy files from a remote host (guest). + + @param host: Hostname or IP address + @param username: Username (if required) + @param password: Password (if required) + @param local_path: Path on the local machine where we are copying from + @param remote_path: Path on the remote machine where we are copying to + @param log_filename: If specified, log all output to this file + @param timeout: The time duration (in seconds) to wait for the transfer + to complete. + @raise: Whatever remote_scp() raises + """ + command = ("scp -v -o UserKnownHostsFile=/dev/null " + "-o PreferredAuthentications=password -r -P %s %s@%s:%s %s" % + (port, username, host, remote_path, local_path)) + remote_scp(command, password, log_filename, timeout) + + +def copy_files_to(address, client, username, password, port, local_path, + remote_path, log_filename=None, verbose=False, timeout=600): + """ + Copy files to a remote host (guest) using the selected client. + + @param client: Type of transfer client + @param username: Username (if required) + @param password: Password (if requried) + @param local_path: Path on the local machine where we are copying from + @param remote_path: Path on the remote machine where we are copying to + @param address: Address of remote host(guest) + @param log_filename: If specified, log all output to this file (SCP only) + @param verbose: If True, log some stats using logging.debug (RSS only) + @param timeout: The time duration (in seconds) to wait for the transfer to + complete. + @raise: Whatever remote_scp() raises + """ + if client == "scp": + scp_to_remote(address, port, username, password, local_path, + remote_path, log_filename, timeout) + elif client == "rss": + log_func = None + if verbose: + log_func = logging.debug + c = rss_client.FileUploadClient(address, port, log_func) + c.upload(local_path, remote_path, timeout) + c.close() + + +def copy_files_from(address, client, username, password, port, remote_path, + local_path, log_filename=None, verbose=False, timeout=600): + """ + Copy files from a remote host (guest) using the selected client. + + @param client: Type of transfer client + @param username: Username (if required) + @param password: Password (if requried) + @param remote_path: Path on the remote machine where we are copying from + @param local_path: Path on the local machine where we are copying to + @param address: Address of remote host(guest) + @param log_filename: If specified, log all output to this file (SCP only) + @param verbose: If True, log some stats using logging.debug (RSS only) + @param timeout: The time duration (in seconds) to wait for the transfer to + complete. + @raise: Whatever remote_scp() raises + """ + if client == "scp": + scp_from_remote(address, port, username, password, remote_path, + local_path, log_filename, timeout) + elif client == "rss": + log_func = None + if verbose: + log_func = logging.debug + c = rss_client.FileDownloadClient(address, port, log_func) + c.download(remote_path, local_path, timeout) + c.close() + + +# The following are utility functions related to ports. + +def is_port_free(port, address): + """ + Return True if the given port is available for use. + + @param port: Port number + """ + try: + s = socket.socket() + #s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if address == "localhost": + s.bind(("localhost", port)) + free = True + else: + s.connect((address, port)) + free = False + except socket.error: + if address == "localhost": + free = False + else: + free = True + s.close() + return free + + +def find_free_port(start_port, end_port, address="localhost"): + """ + Return a host free port in the range [start_port, end_port]. + + @param start_port: First port that will be checked. + @param end_port: Port immediately after the last one that will be checked. + """ + for i in range(start_port, end_port): + if is_port_free(i, address): + return i + return None + + +def find_free_ports(start_port, end_port, count, address="localhost"): + """ + Return count of host free ports in the range [start_port, end_port]. + + @count: Initial number of ports known to be free in the range. + @param start_port: First port that will be checked. + @param end_port: Port immediately after the last one that will be checked. + """ + ports = [] + i = start_port + while i < end_port and count > 0: + if is_port_free(i, address): + ports.append(i) + count -= 1 + i += 1 + return ports + + +# An easy way to log lines to files when the logging system can't be used + +_open_log_files = {} +_log_file_dir = "/tmp" + + +def log_line(filename, line): + """ + Write a line to a file. '\n' is appended to the line. + + @param filename: Path of file to write to, either absolute or relative to + the dir set by set_log_file_dir(). + @param line: Line to write. + """ + global _open_log_files, _log_file_dir + if filename not in _open_log_files: + path = get_path(_log_file_dir, filename) + try: + os.makedirs(os.path.dirname(path)) + except OSError: + pass + _open_log_files[filename] = open(path, "w") + timestr = time.strftime("%Y-%m-%d %H:%M:%S") + _open_log_files[filename].write("%s: %s\n" % (timestr, line)) + _open_log_files[filename].flush() + + +def set_log_file_dir(dir): + """ + Set the base directory for log files created by log_line(). + + @param dir: Directory for log files. + """ + global _log_file_dir + _log_file_dir = dir + + +# The following are miscellaneous utility functions. + +def get_path(base_path, user_path): + """ + Translate a user specified path to a real path. + If user_path is relative, append it to base_path. + If user_path is absolute, return it as is. + + @param base_path: The base path of relative user specified paths. + @param user_path: The user specified path. + """ + if os.path.isabs(user_path): + return user_path + else: + return os.path.join(base_path, user_path) + + +def generate_random_string(length): + """ + Return a random string using alphanumeric characters. + + @length: length of the string that will be generated. + """ + r = random.SystemRandom() + str = "" + chars = string.letters + string.digits + while length > 0: + str += r.choice(chars) + length -= 1 + return str + +def generate_random_id(): + """ + Return a random string suitable for use as a qemu id. + """ + return "id" + generate_random_string(6) + + +def generate_tmp_file_name(file, ext=None, dir='/tmp/'): + """ + Returns a temporary file name. The file is not created. + """ + while True: + file_name = (file + '-' + time.strftime("%Y%m%d-%H%M%S-") + + generate_random_string(4)) + if ext: + file_name += '.' + ext + file_name = os.path.join(dir, file_name) + if not os.path.exists(file_name): + break + + return file_name + + +def format_str_for_message(str): + """ + Format str so that it can be appended to a message. + If str consists of one line, prefix it with a space. + If str consists of multiple lines, prefix it with a newline. + + @param str: string that will be formatted. + """ + lines = str.splitlines() + num_lines = len(lines) + str = "\n".join(lines) + if num_lines == 0: + return "" + elif num_lines == 1: + return " " + str + else: + return "\n" + str + + +def wait_for(func, timeout, first=0.0, step=1.0, text=None): + """ + If func() evaluates to True before timeout expires, return the + value of func(). Otherwise return None. + + @brief: Wait until func() evaluates to True. + + @param timeout: Timeout in seconds + @param first: Time to sleep before first attempt + @param steps: Time to sleep between attempts in seconds + @param text: Text to print while waiting, for debug purposes + """ + start_time = time.time() + end_time = time.time() + timeout + + time.sleep(first) + + while time.time() < end_time: + if text: + logging.debug("%s (%f secs)", text, (time.time() - start_time)) + + output = func() + if output: + return output + + time.sleep(step) + + logging.debug("Timeout elapsed") + return None + + +def get_hash_from_file(hash_path, dvd_basename): + """ + Get the a hash from a given DVD image from a hash file + (Hash files are usually named MD5SUM or SHA1SUM and are located inside the + download directories of the DVDs) + + @param hash_path: Local path to a hash file. + @param cd_image: Basename of a CD image + """ + hash_file = open(hash_path, 'r') + for line in hash_file.readlines(): + if dvd_basename in line: + return line.split()[0] + + +def run_tests(parser, job): + """ + Runs the sequence of KVM tests based on the list of dictionaries + generated by the configuration system, handling dependencies. + + @param parser: Config parser object. + @param job: Autotest job object. + + @return: True, if all tests ran passed, False if any of them failed. + """ + for i, d in enumerate(parser.get_dicts()): + logging.info("Test %4d: %s" % (i + 1, d["shortname"])) + + status_dict = {} + failed = False + + for dict in parser.get_dicts(): + if dict.get("skip") == "yes": + continue + dependencies_satisfied = True + for dep in dict.get("dep"): + for test_name in status_dict.keys(): + if not dep in test_name: + continue + if not status_dict[test_name]: + dependencies_satisfied = False + break + if dependencies_satisfied: + test_iterations = int(dict.get("iterations", 1)) + test_tag = dict.get("shortname") + # Setting up profilers during test execution. + profilers = dict.get("profilers", "").split() + for profiler in profilers: + job.profilers.add(profiler) + + # We need only one execution, profiled, hence we're passing + # the profile_only parameter to job.run_test(). + current_status = job.run_test("kvm", params=dict, tag=test_tag, + iterations=test_iterations, + profile_only= bool(profilers) or None) + + for profiler in profilers: + job.profilers.delete(profiler) + + if not current_status: + failed = True + else: + current_status = False + status_dict[dict.get("name")] = current_status + + return not failed + + +def create_report(report_dir, results_dir): + """ + Creates a neatly arranged HTML results report in the results dir. + + @param report_dir: Directory where the report script is located. + @param results_dir: Directory where the results will be output. + """ + reporter = os.path.join(report_dir, 'html_report.py') + html_file = os.path.join(results_dir, 'results.html') + os.system('%s -r %s -f %s -R' % (reporter, results_dir, html_file)) + + +def get_full_pci_id(pci_id): + """ + Get full PCI ID of pci_id. + + @param pci_id: PCI ID of a device. + """ + cmd = "lspci -D | awk '/%s/ {print $1}'" % pci_id + status, full_id = commands.getstatusoutput(cmd) + if status != 0: + return None + return full_id + + +def get_vendor_from_pci_id(pci_id): + """ + Check out the device vendor ID according to pci_id. + + @param pci_id: PCI ID of a device. + """ + cmd = "lspci -n | awk '/%s/ {print $3}'" % pci_id + return re.sub(":", " ", commands.getoutput(cmd)) + + +class Thread(threading.Thread): + """ + Run a function in a background thread. + """ + def __init__(self, target, args=(), kwargs={}): + """ + Initialize the instance. + + @param target: Function to run in the thread. + @param args: Arguments to pass to target. + @param kwargs: Keyword arguments to pass to target. + """ + threading.Thread.__init__(self) + self._target = target + self._args = args + self._kwargs = kwargs + + + def run(self): + """ + Run target (passed to the constructor). No point in calling this + function directly. Call start() to make this function run in a new + thread. + """ + self._e = None + self._retval = None + try: + try: + self._retval = self._target(*self._args, **self._kwargs) + except: + self._e = sys.exc_info() + raise + finally: + # Avoid circular references (start() may be called only once so + # it's OK to delete these) + del self._target, self._args, self._kwargs + + + def join(self, timeout=None, suppress_exception=False): + """ + Join the thread. If target raised an exception, re-raise it. + Otherwise, return the value returned by target. + + @param timeout: Timeout value to pass to threading.Thread.join(). + @param suppress_exception: If True, don't re-raise the exception. + """ + threading.Thread.join(self, timeout) + try: + if self._e: + if not suppress_exception: + # Because the exception was raised in another thread, we + # need to explicitly insert the current context into it + s = error.exception_context(self._e[1]) + s = error.join_contexts(error.get_context(), s) + error.set_exception_context(self._e[1], s) + raise self._e[0], self._e[1], self._e[2] + else: + return self._retval + finally: + # Avoid circular references (join() may be called multiple times + # so we can't delete these) + self._e = None + self._retval = None + + +def parallel(targets): + """ + Run multiple functions in parallel. + + @param targets: A sequence of tuples or functions. If it's a sequence of + tuples, each tuple will be interpreted as (target, args, kwargs) or + (target, args) or (target,) depending on its length. If it's a + sequence of functions, the functions will be called without + arguments. + @return: A list of the values returned by the functions called. + """ + threads = [] + for target in targets: + if isinstance(target, tuple) or isinstance(target, list): + t = Thread(*target) + else: + t = Thread(target) + threads.append(t) + t.start() + return [t.join() for t in threads] + + +class KvmLoggingConfig(logging_config.LoggingConfig): + """ + Used with the sole purpose of providing convenient logging setup + for the KVM test auxiliary programs. + """ + def configure_logging(self, results_dir=None, verbose=False): + super(KvmLoggingConfig, self).configure_logging(use_console=True, + verbose=verbose) + + +class PciAssignable(object): + """ + Request PCI assignable devices on host. It will check whether to request + PF (physical Functions) or VF (Virtual Functions). + """ + def __init__(self, type="vf", driver=None, driver_option=None, + names=None, devices_requested=None): + """ + Initialize parameter 'type' which could be: + vf: Virtual Functions + pf: Physical Function (actual hardware) + mixed: Both includes VFs and PFs + + If pass through Physical NIC cards, we need to specify which devices + to be assigned, e.g. 'eth1 eth2'. + + If pass through Virtual Functions, we need to specify how many vfs + are going to be assigned, e.g. passthrough_count = 8 and max_vfs in + config file. + + @param type: PCI device type. + @param driver: Kernel module for the PCI assignable device. + @param driver_option: Module option to specify the maximum number of + VFs (eg 'max_vfs=7') + @param names: Physical NIC cards correspondent network interfaces, + e.g.'eth1 eth2 ...' + @param devices_requested: Number of devices being requested. + """ + self.type = type + self.driver = driver + self.driver_option = driver_option + if names: + self.name_list = names.split() + if devices_requested: + self.devices_requested = int(devices_requested) + else: + self.devices_requested = None + + + def _get_pf_pci_id(self, name, search_str): + """ + Get the PF PCI ID according to name. + + @param name: Name of the PCI device. + @param search_str: Search string to be used on lspci. + """ + cmd = "ethtool -i %s | awk '/bus-info/ {print $2}'" % name + s, pci_id = commands.getstatusoutput(cmd) + if not (s or "Cannot get driver information" in pci_id): + return pci_id[5:] + cmd = "lspci | awk '/%s/ {print $1}'" % search_str + pci_ids = [id for id in commands.getoutput(cmd).splitlines()] + nic_id = int(re.search('[0-9]+', name).group(0)) + if (len(pci_ids) - 1) < nic_id: + return None + return pci_ids[nic_id] + + + def _release_dev(self, pci_id): + """ + Release a single PCI device. + + @param pci_id: PCI ID of a given PCI device. + """ + base_dir = "/sys/bus/pci" + full_id = get_full_pci_id(pci_id) + vendor_id = get_vendor_from_pci_id(pci_id) + drv_path = os.path.join(base_dir, "devices/%s/driver" % full_id) + if 'pci-stub' in os.readlink(drv_path): + cmd = "echo '%s' > %s/new_id" % (vendor_id, drv_path) + if os.system(cmd): + return False + + stub_path = os.path.join(base_dir, "drivers/pci-stub") + cmd = "echo '%s' > %s/unbind" % (full_id, stub_path) + if os.system(cmd): + return False + + driver = self.dev_drivers[pci_id] + cmd = "echo '%s' > %s/bind" % (full_id, driver) + if os.system(cmd): + return False + + return True + + + def get_vf_devs(self): + """ + Catch all VFs PCI IDs. + + @return: List with all PCI IDs for the Virtual Functions avaliable + """ + if not self.sr_iov_setup(): + return [] + + cmd = "lspci | awk '/Virtual Function/ {print $1}'" + return commands.getoutput(cmd).split() + + + def get_pf_devs(self): + """ + Catch all PFs PCI IDs. + + @return: List with all PCI IDs for the physical hardware requested + """ + pf_ids = [] + for name in self.name_list: + pf_id = self._get_pf_pci_id(name, "Ethernet") + if not pf_id: + continue + pf_ids.append(pf_id) + return pf_ids + + + def get_devs(self, count): + """ + Check out all devices' PCI IDs according to their name. + + @param count: count number of PCI devices needed for pass through + @return: a list of all devices' PCI IDs + """ + if self.type == "vf": + vf_ids = self.get_vf_devs() + elif self.type == "pf": + vf_ids = self.get_pf_devs() + elif self.type == "mixed": + vf_ids = self.get_vf_devs() + vf_ids.extend(self.get_pf_devs()) + return vf_ids[0:count] + + + def get_vfs_count(self): + """ + Get VFs count number according to lspci. + """ + # FIXME: Need to think out a method of identify which + # 'virtual function' belongs to which physical card considering + # that if the host has more than one 82576 card. PCI_ID? + cmd = "lspci | grep 'Virtual Function' | wc -l" + return int(commands.getoutput(cmd)) + + + def check_vfs_count(self): + """ + Check VFs count number according to the parameter driver_options. + """ + # Network card 82576 has two network interfaces and each can be + # virtualized up to 7 virtual functions, therefore we multiply + # two for the value of driver_option 'max_vfs'. + expected_count = int((re.findall("(\d)", self.driver_option)[0])) * 2 + return (self.get_vfs_count == expected_count) + + + def is_binded_to_stub(self, full_id): + """ + Verify whether the device with full_id is already binded to pci-stub. + + @param full_id: Full ID for the given PCI device + """ + base_dir = "/sys/bus/pci" + stub_path = os.path.join(base_dir, "drivers/pci-stub") + if os.path.exists(os.path.join(stub_path, full_id)): + return True + return False + + + def sr_iov_setup(self): + """ + Ensure the PCI device is working in sr_iov mode. + + Check if the PCI hardware device drive is loaded with the appropriate, + parameters (number of VFs), and if it's not, perform setup. + + @return: True, if the setup was completed successfuly, False otherwise. + """ + re_probe = False + s, o = commands.getstatusoutput('lsmod | grep %s' % self.driver) + if s: + re_probe = True + elif not self.check_vfs_count(): + os.system("modprobe -r %s" % self.driver) + re_probe = True + else: + return True + + # Re-probe driver with proper number of VFs + if re_probe: + cmd = "modprobe %s %s" % (self.driver, self.driver_option) + logging.info("Loading the driver '%s' with option '%s'", + self.driver, self.driver_option) + s, o = commands.getstatusoutput(cmd) + if s: + return False + return True + + + def request_devs(self): + """ + Implement setup process: unbind the PCI device and then bind it + to the pci-stub driver. + + @return: a list of successfully requested devices' PCI IDs. + """ + base_dir = "/sys/bus/pci" + stub_path = os.path.join(base_dir, "drivers/pci-stub") + + self.pci_ids = self.get_devs(self.devices_requested) + logging.debug("The following pci_ids were found: %s", self.pci_ids) + requested_pci_ids = [] + self.dev_drivers = {} + + # Setup all devices specified for assignment to guest + for pci_id in self.pci_ids: + full_id = get_full_pci_id(pci_id) + if not full_id: + continue + drv_path = os.path.join(base_dir, "devices/%s/driver" % full_id) + dev_prev_driver = os.path.realpath(os.path.join(drv_path, + os.readlink(drv_path))) + self.dev_drivers[pci_id] = dev_prev_driver + + # Judge whether the device driver has been binded to stub + if not self.is_binded_to_stub(full_id): + logging.debug("Binding device %s to stub", full_id) + vendor_id = get_vendor_from_pci_id(pci_id) + stub_new_id = os.path.join(stub_path, 'new_id') + unbind_dev = os.path.join(drv_path, 'unbind') + stub_bind = os.path.join(stub_path, 'bind') + + info_write_to_files = [(vendor_id, stub_new_id), + (full_id, unbind_dev), + (full_id, stub_bind)] + + for content, file in info_write_to_files: + try: + utils.open_write_close(file, content) + except IOError: + logging.debug("Failed to write %s to file %s", content, + file) + continue + + if not self.is_binded_to_stub(full_id): + logging.error("Binding device %s to stub failed", pci_id) + continue + else: + logging.debug("Device %s already binded to stub", pci_id) + requested_pci_ids.append(pci_id) + self.pci_ids = requested_pci_ids + return self.pci_ids + + + def release_devs(self): + """ + Release all PCI devices currently assigned to VMs back to the + virtualization host. + """ + try: + for pci_id in self.dev_drivers: + if not self._release_dev(pci_id): + logging.error("Failed to release device %s to host", pci_id) + else: + logging.info("Released device %s successfully", pci_id) + except: + return + + +class KojiDownloader(object): + """ + Stablish a connection with the build system, either koji or brew. + + This class provides a convenience methods to retrieve packages hosted on + the build system. + """ + def __init__(self, cmd): + """ + Verifies whether the system has koji or brew installed, then loads + the configuration file that will be used to download the files. + + @param cmd: Command name, either 'brew' or 'koji'. It is important + to figure out the appropriate configuration used by the + downloader. + @param dst_dir: Destination dir for the packages. + """ + if not KOJI_INSTALLED: + raise ValueError('No koji/brew installed on the machine') + + if os.path.isfile(cmd): + koji_cmd = cmd + else: + koji_cmd = os_dep.command(cmd) + + logging.debug("Found %s as the buildsystem interface", koji_cmd) + + config_map = {'/usr/bin/koji': '/etc/koji.conf', + '/usr/bin/brew': '/etc/brewkoji.conf'} + + try: + config_file = config_map[koji_cmd] + except IndexError: + raise ValueError('Could not find config file for %s' % koji_cmd) + + base_name = os.path.basename(koji_cmd) + if os.access(config_file, os.F_OK): + f = open(config_file) + config = ConfigParser.ConfigParser() + config.readfp(f) + f.close() + else: + raise IOError('Configuration file %s missing or with wrong ' + 'permissions' % config_file) + + if config.has_section(base_name): + self.koji_options = {} + session_options = {} + server = None + for name, value in config.items(base_name): + if name in ('user', 'password', 'debug_xmlrpc', 'debug'): + session_options[name] = value + self.koji_options[name] = value + self.session = koji.ClientSession(self.koji_options['server'], + session_options) + else: + raise ValueError('Koji config file %s does not have a %s ' + 'session' % (config_file, base_name)) + + + def get(self, src_package, dst_dir, rfilter=None, tag=None, build=None, + arch=None): + """ + Download a list of packages from the build system. + + This will download all packages originated from source package [package] + with given [tag] or [build] for the architecture reported by the + machine. + + @param src_package: Source package name. + @param dst_dir: Destination directory for the downloaded packages. + @param rfilter: Regexp filter, only download the packages that match + that particular filter. + @param tag: Build system tag. + @param build: Build system ID. + @param arch: Package arch. Useful when you want to download noarch + packages. + + @return: List of paths with the downloaded rpm packages. + """ + if build and build.isdigit(): + build = int(build) + + if tag and build: + logging.info("Both tag and build parameters provided, ignoring tag " + "parameter...") + + if not tag and not build: + raise ValueError("Koji install selected but neither koji_tag " + "nor koji_build parameters provided. Please " + "provide an appropriate tag or build name.") + + if not build: + builds = self.session.listTagged(tag, latest=True, inherit=True, + package=src_package) + if not builds: + raise ValueError("Tag %s has no builds of %s" % (tag, + src_package)) + info = builds[0] + else: + info = self.session.getBuild(build) + + if info is None: + raise ValueError('No such brew/koji build: %s' % build) + + if arch is None: + arch = utils.get_arch() + + rpms = self.session.listRPMs(buildID=info['id'], + arches=arch) + if not rpms: + raise ValueError("No %s packages available for %s" % + arch, koji.buildLabel(info)) + + rpm_paths = [] + for rpm in rpms: + rpm_name = koji.pathinfo.rpm(rpm) + url = ("%s/%s/%s/%s/%s" % (self.koji_options['pkgurl'], + info['package_name'], + info['version'], info['release'], + rpm_name)) + if rfilter: + filter_regexp = re.compile(rfilter, re.IGNORECASE) + if filter_regexp.match(os.path.basename(rpm_name)): + download = True + else: + download = False + else: + download = True + + if download: + r = utils.get_file(url, + os.path.join(dst_dir, os.path.basename(url))) + rpm_paths.append(r) + + return rpm_paths + + +def umount(src, mount_point, type): + """ + Umount the src mounted in mount_point. + + @src: mount source + @mount_point: mount point + @type: file system type + """ + + mount_string = "%s %s %s" % (src, mount_point, type) + if mount_string in file("/etc/mtab").read(): + umount_cmd = "umount %s" % mount_point + try: + utils.system(umount_cmd) + return True + except error.CmdError: + return False + else: + logging.debug("%s is not mounted under %s", src, mount_point) + return True + + +def mount(src, mount_point, type, perm="rw"): + """ + Mount the src into mount_point of the host. + + @src: mount source + @mount_point: mount point + @type: file system type + @perm: mount premission + """ + umount(src, mount_point, type) + mount_string = "%s %s %s %s" % (src, mount_point, type, perm) + + if mount_string in file("/etc/mtab").read(): + logging.debug("%s is already mounted in %s with %s", + src, mount_point, perm) + return True + + mount_cmd = "mount -t %s %s %s -o %s" % (type, src, mount_point, perm) + try: + utils.system(mount_cmd) + except error.CmdError: + return False + + logging.debug("Verify the mount through /etc/mtab") + if mount_string in file("/etc/mtab").read(): + logging.debug("%s is successfully mounted", src) + return True + else: + logging.error("Can't find mounted NFS share - /etc/mtab contents \n%s", + file("/etc/mtab").read()) + return False diff --git a/client/virt/virt_vm.py b/client/virt/virt_vm.py new file mode 100644 index 0000000..ece90c8 --- /dev/null +++ b/client/virt/virt_vm.py @@ -0,0 +1,298 @@ +import os, logging +from autotest_lib.client.common_lib import error +from autotest_lib.client.bin import utils +import virt_utils, kvm_vm + +class VMError(Exception): + pass + + +class VMCreateError(VMError): + def __init__(self, cmd, status, output): + VMError.__init__(self, cmd, status, output) + self.cmd = cmd + self.status = status + self.output = output + + def __str__(self): + return ("VM creation command failed: %r (status: %s, " + "output: %r)" % (self.cmd, self.status, self.output)) + + +class VMHashMismatchError(VMError): + def __init__(self, actual, expected): + VMError.__init__(self, actual, expected) + self.actual_hash = actual + self.expected_hash = expected + + def __str__(self): + return ("CD image hash (%s) differs from expected one (%s)" % + (self.actual_hash, self.expected_hash)) + + +class VMImageMissingError(VMError): + def __init__(self, filename): + VMError.__init__(self, filename) + self.filename = filename + + def __str__(self): + return "CD image file not found: %r" % self.filename + + +class VMImageCheckError(VMError): + def __init__(self, filename): + VMError.__init__(self, filename) + self.filename = filename + + def __str__(self): + return "Errors found on image: %r" % self.filename + + +class VMBadPATypeError(VMError): + def __init__(self, pa_type): + VMError.__init__(self, pa_type) + self.pa_type = pa_type + + def __str__(self): + return "Unsupported PCI assignable type: %r" % self.pa_type + + +class VMPAError(VMError): + def __init__(self, pa_type): + VMError.__init__(self, pa_type) + self.pa_type = pa_type + + def __str__(self): + return ("No PCI assignable devices could be assigned " + "(pci_assignable=%r)" % self.pa_type) + + +class VMPostCreateError(VMError): + def __init__(self, cmd, output): + VMError.__init__(self, cmd, output) + self.cmd = cmd + self.output = output + + +class VMHugePageError(VMPostCreateError): + def __str__(self): + return ("Cannot allocate hugepage memory (command: %r, " + "output: %r)" % (self.cmd, self.output)) + + +class VMKVMInitError(VMPostCreateError): + def __str__(self): + return ("Cannot initialize KVM (command: %r, output: %r)" % + (self.cmd, self.output)) + + +class VMDeadError(VMError): + def __init__(self, status, output): + VMError.__init__(self, status, output) + self.status = status + self.output = output + + def __str__(self): + return ("VM process is dead (status: %s, output: %r)" % + (self.status, self.output)) + + +class VMAddressError(VMError): + pass + + +class VMPortNotRedirectedError(VMAddressError): + def __init__(self, port): + VMAddressError.__init__(self, port) + self.port = port + + def __str__(self): + return "Port not redirected: %s" % self.port + + +class VMAddressVerificationError(VMAddressError): + def __init__(self, mac, ip): + VMAddressError.__init__(self, mac, ip) + self.mac = mac + self.ip = ip + + def __str__(self): + return ("Cannot verify MAC-IP address mapping using arping: " + "%s ---> %s" % (self.mac, self.ip)) + + +class VMMACAddressMissingError(VMAddressError): + def __init__(self, nic_index): + VMAddressError.__init__(self, nic_index) + self.nic_index = nic_index + + def __str__(self): + return "No MAC address defined for NIC #%s" % self.nic_index + + +class VMIPAddressMissingError(VMAddressError): + def __init__(self, mac): + VMAddressError.__init__(self, mac) + self.mac = mac + + def __str__(self): + return "Cannot find IP address for MAC address %s" % self.mac + + +class VMMigrateError(VMError): + pass + + +class VMMigrateTimeoutError(VMMigrateError): + pass + + +class VMMigrateCancelError(VMMigrateError): + pass + + +class VMMigrateFailedError(VMMigrateError): + pass + + +class VMMigrateStateMismatchError(VMMigrateError): + def __init__(self, src_hash, dst_hash): + VMMigrateError.__init__(self, src_hash, dst_hash) + self.src_hash = src_hash + self.dst_hash = dst_hash + + def __str__(self): + return ("Mismatch of VM state before and after migration (%s != %s)" % + (self.src_hash, self.dst_hash)) + + +class VMRebootError(VMError): + pass + + +def get_image_filename(params, root_dir): + """ + Generate an image path from params and root_dir. + + @param params: Dictionary containing the test parameters. + @param root_dir: Base directory for relative filenames. + + @note: params should contain: + image_name -- the name of the image file, without extension + image_format -- the format of the image (qcow2, raw etc) + """ + image_name = params.get("image_name", "image") + image_format = params.get("image_format", "qcow2") + if params.get("image_raw_device") == "yes": + return image_name + image_filename = "%s.%s" % (image_name, image_format) + image_filename = virt_utils.get_path(root_dir, image_filename) + return image_filename + + +def create_image(params, root_dir): + """ + Create an image using qemu_image. + + @param params: Dictionary containing the test parameters. + @param root_dir: Base directory for relative filenames. + + @note: params should contain: + image_name -- the name of the image file, without extension + image_format -- the format of the image (qcow2, raw etc) + image_size -- the requested size of the image (a string + qemu-img can understand, such as '10G') + """ + qemu_img_cmd = virt_utils.get_path(root_dir, params.get("qemu_img_binary", + "qemu-img")) + qemu_img_cmd += " create" + + format = params.get("image_format", "qcow2") + qemu_img_cmd += " -f %s" % format + + image_filename = get_image_filename(params, root_dir) + qemu_img_cmd += " %s" % image_filename + + size = params.get("image_size", "10G") + qemu_img_cmd += " %s" % size + + utils.system(qemu_img_cmd) + logging.info("Image created in %r", image_filename) + return image_filename + + +def remove_image(params, root_dir): + """ + Remove an image file. + + @param params: A dict + @param root_dir: Base directory for relative filenames. + + @note: params should contain: + image_name -- the name of the image file, without extension + image_format -- the format of the image (qcow2, raw etc) + """ + image_filename = get_image_filename(params, root_dir) + logging.debug("Removing image file %s...", image_filename) + if os.path.exists(image_filename): + os.unlink(image_filename) + else: + logging.debug("Image file %s not found") + + +def check_image(params, root_dir): + """ + Check an image using the appropriate tools for each virt backend. + + @param params: Dictionary containing the test parameters. + @param root_dir: Base directory for relative filenames. + + @note: params should contain: + image_name -- the name of the image file, without extension + image_format -- the format of the image (qcow2, raw etc) + + @raise VMImageCheckError: In case qemu-img check fails on the image. + """ + vm_type = params.get("vm_type") + if vm_type == 'kvm': + image_filename = get_image_filename(params, root_dir) + logging.debug("Checking image file %s...", image_filename) + qemu_img_cmd = virt_utils.get_path(root_dir, + params.get("qemu_img_binary", "qemu-img")) + image_is_qcow2 = params.get("image_format") == 'qcow2' + if os.path.exists(image_filename) and image_is_qcow2: + # Verifying if qemu-img supports 'check' + q_result = utils.run(qemu_img_cmd, ignore_status=True) + q_output = q_result.stdout + check_img = True + if not "check" in q_output: + logging.error("qemu-img does not support 'check', " + "skipping check...") + check_img = False + if not "info" in q_output: + logging.error("qemu-img does not support 'info', " + "skipping check...") + check_img = False + if check_img: + try: + utils.system("%s info %s" % (qemu_img_cmd, image_filename)) + except error.CmdError: + logging.error("Error getting info from image %s", + image_filename) + try: + utils.system("%s check %s" % (qemu_img_cmd, image_filename)) + except error.CmdError: + raise VMImageCheckError(image_filename) + + else: + if not os.path.exists(image_filename): + logging.debug("Image file %s not found, skipping check...", + image_filename) + elif not image_is_qcow2: + logging.debug("Image file %s not qcow2, skipping check...", + image_filename) + + +def instantiate_vm(vm_type, **kwargs): + if vm_type == 'kvm': + return kvm_vm.VM(**kwargs) -- 1.7.4