[PATCH 1/5] KVM test: Include the preprocessing param migration_mode

[PATCH 1/5] KVM test: Include the preprocessing param migration_mode

[Lucas Meneghel Rodrigues]

In order to include multi-host migration, add a pre-processor
parameter that will make the test start a VM in incoming
migration mode:

migration_mode = tcp

Will make the preprocessor start the VM in tcp -incoming mode.

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/kvm_preprocessing.py |   16 +++++++++++++---
 1 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/client/tests/kvm/kvm_preprocessing.py b/client/tests/kvm/kvm_preprocessing.py
index 5ec38fb..d2f94cd 100644
--- a/client/tests/kvm/kvm_preprocessing.py
+++ b/client/tests/kvm/kvm_preprocessing.py
@@ -60,6 +60,8 @@ def preprocess_vm(test, params, env, name):
 
     start_vm = False
 
+    migration_mode = params.get("migration_mode", None)
+
     if params.get("restart_vm") == "yes":
         logging.debug("'restart_vm' specified; (re)starting VM...")
         start_vm = True
@@ -72,11 +74,19 @@ def preprocess_vm(test, params, env, name):
             logging.debug("VM's qemu command differs from requested one; "
                           "restarting it...")
             start_vm = True
+    elif migration_mode is not None:
+        logging.debug("Starting VM on migration incoming mode...")
+        start_vm = True
 
     if start_vm:
-        # Start the VM (or restart it if it's already up)
-        if not vm.create(name, params, test.bindir):
-            raise error.TestError("Could not start VM")
+        if migration_mode is not None:
+            if not vm.create(name, params, test.bindir,
+                             migration_mode=migration_mode):
+                raise error.TestError("Could not start VM for migration")
+        else:
+            # Start the VM (or restart it if it's already up)
+            if not vm.create(name, params, test.bindir):
+                raise error.TestError("Could not start VM")
     else:
         # Don't start the VM, just update its params
         vm.params = params
-- 
1.7.3.4

[PATCH 2/5] KVM test: Introduce migration_multi_host test

[Lucas Meneghel Rodrigues]

This test will carry out the migration between guests in
two different hosts. Depending on the test 'role', different
actions will be taken, and those actions will be synchronized
through the autotest barrier mechanism.

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/tests/migration_multi_host.py |  108 ++++++++++++++++++++++++
 client/tests/kvm/tests_base.cfg.sample         |   13 +++
 2 files changed, 121 insertions(+), 0 deletions(-)
 create mode 100644 client/tests/kvm/tests/migration_multi_host.py

diff --git a/client/tests/kvm/tests/migration_multi_host.py b/client/tests/kvm/tests/migration_multi_host.py
new file mode 100644
index 0000000..15af0c8
--- /dev/null
+++ b/client/tests/kvm/tests/migration_multi_host.py
@@ -0,0 +1,108 @@
+import logging, time, socket
+from autotest_lib.client.common_lib import error
+import kvm_subprocess, kvm_test_utils, kvm_utils
+
+
+def run_migration_multi_host(test, params, env):
+    """
+    KVM multi-host migration test:
+
+    Migration execution progress:
+
+    source host                       dest host
+    ----------------------------------------------------------------------------
+    log into guest
+    ----------------------------------------------------------------------------
+    start socket server
+
+    wait 30 secs -------------------- wait login_timeout+30 secs ---------------
+
+    accept connection                 connect to socket server,send mig_port
+    ----------------------------------------------------------------------------
+    start migration
+
+    wait 30 secs -------------------- wait mig_timeout+30 secs -----------------
+
+    try to log into migrated guest    check VM's status via monitor cmd
+    ----------------------------------------------------------------------------
+
+    @param test: kvm test object.
+    @param params: Dictionary with test parameters.
+    @param env: Dictionary with the test environment.
+    """
+    def guest_active(vm):
+        o = vm.monitor.info("status")
+        if isinstance(o, str):
+            return "status: running" in o
+        else:
+            return o.get("status") == "running"
+
+    vm = kvm_test_utils.get_living_vm(env, params.get("main_vm"))
+    login_timeout = int(params.get("login_timeout", 360))
+    role = params.get("role")
+    srchost = params.get("srchost")
+    dsthost = params.get("dsthost")
+    mig_timeout = int(params.get("mig_timeout"))
+    # Port used to communicate info between source and destination
+    comm_port = int(params.get("comm_port", 12324))
+    regain_ip_cmd = params.get("regain_ip_cmd", "dhclient")
+    if role == 'source':
+        session = kvm_test_utils.wait_for_login(vm, timeout=login_timeout)
+
+        # Listen on a port to get the migration port received from
+        # dest machine
+        s_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s_socket.bind(('', comm_port))
+        s_socket.listen(1)
+
+        # Wait 30 seconds for source and dest to reach this point
+        test.job.barrier(srchost,'socket_started', 30).rendezvous(srchost,
+                                                                  dsthost)
+
+        c_socket, addr = s_socket.accept()
+        mig_port = int(c_socket.recv(6))
+        logging.info("Received from destination the migration port %s",
+                     mig_port)
+        c_socket.close()
+
+        logging.info("Start migrating now...")
+        kvm_test_utils.migrate(vm=vm, dest_host=dsthost, mig_port=mig_port,
+                               env=env)
+
+        # Wait up to 30 seconds for dest to reach this point
+        test.job.barrier(srchost, 'mig_finished', 30).rendezvous(srchost,
+                                                                 dsthost)
+
+    elif role == 'destination':
+        # Wait up to login_timeout + 30 seconds for the source to
+        # reach this point
+        test.job.barrier(dsthost, 'socket_started',
+                         login_timeout + 30).rendezvous(srchost,
+                                                        dsthost)
+
+        c_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        c_socket.connect((srchost, comm_port))
+        logging.info("Communicating to source migration port %s",
+                     vm.migration_port)
+        c_socket.send("%d" % vm.migration_port)
+        c_socket.close()
+
+        # Wait up to mig_timeout + 30 seconds for the source to
+        # reach this point: migration finished
+        test.job.barrier(dsthost, 'mig_finished',
+                         mig_timeout + 30).rendezvous(srchost,
+                                                      dsthost)
+
+        if not guest_active(vm):
+            raise error.TestFail("Guest not active after migration")
+
+        logging.info("Migrated guest appears to be running")
+
+        # Log into the guest again
+        logging.info("Logging into migrated guest after migration...")
+        session_serial = kvm_test_utils.wait_for_login(vm, timeout=login_timeout, serial=True)
+        session_serial.cmd(regain_ip_cmd)
+        session = kvm_test_utils.wait_for_login(vm, timeout=login_timeout)
+
+    else:
+        raise error.TestError('Invalid role specified')
diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample
index 9ef29d7..e5ceaf7 100644
--- a/client/tests/kvm/tests_base.cfg.sample
+++ b/client/tests/kvm/tests_base.cfg.sample
@@ -159,6 +159,19 @@ variants:
                 iterations = 1
                 type = migration_with_file_transfer
 
+    - migrate_multi_host:      install setup unattended_install.cdrom
+        type = migration_multi_host
+        migration_test_command = help
+        migration_bg_command = "cd /tmp; nohup tcpdump -q -t ip host localhost"
+        migration_bg_check_command = pgrep tcpdump
+        migration_bg_kill_command = pkill tcpdump
+        kill_vm_on_error = yes
+        iterations = 2
+        used_mem = 1024
+        mig_timeout = 3600
+        comm_port = 13234
+        regain_ip_cmd = dhclient
+
     - boot_savevm: install setup unattended_install.cdrom
         type = boot_savevm
         savevm_delay = 0.3
-- 
1.7.3.4

[PATCH 3/5] KVM test: Add migration server control file

[Lucas Meneghel Rodrigues]

Introduce the server migration control file, that makes
possible to run the multi host migration. It parses the
base config file, restricts it with appropriate
parameters, generates the test dicts, modify the
test_dicts so there's a distinction between the
migration roles ('destination' or 'source').

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/migration_control.srv |  122 ++++++++++++++++++++++++++++++++
 1 files changed, 122 insertions(+), 0 deletions(-)
 create mode 100644 client/tests/kvm/migration_control.srv

diff --git a/client/tests/kvm/migration_control.srv b/client/tests/kvm/migration_control.srv
new file mode 100644
index 0000000..16ada36
--- /dev/null
+++ b/client/tests/kvm/migration_control.srv
@@ -0,0 +1,122 @@
+AUTHOR = "Yolkfull Chow <yzhou@redhat.com>"
+TIME = "SHORT"
+NAME = "Migration across multiple hosts"
+TEST_CATEGORY = "Functional"
+TEST_CLASS = "Virtualization"
+TEST_TYPE = "Server"
+DOC = """
+Migrate KVM guest between two hosts. It parses the base config file, restricts
+it with appropriate parameters, generates the test dicts, modify the test_dicts
+so there's a distinction between the migration roles ('dest' or 'source').
+"""
+
+import sys, os, commands, glob, shutil, logging, random
+from autotest_lib.server import utils
+
+# Specify the directory of autotest before you start this test
+AUTOTEST_DIR = '/usr/local/autotest'
+
+# Specify the root directory that on client machines
+rootdir = '/tmp/kvm_autotest_root'
+
+# Make possible to import the KVM test APIs
+KVM_DIR = os.path.join(AUTOTEST_DIR, 'client/tests/kvm')
+sys.path.append(KVM_DIR)
+
+import common, kvm_config
+
+def generate_mac_address():
+    r = random.SystemRandom()
+    mac = "9a:%02x:%02x:%02x:%02x:%02x" % (r.randint(0x00, 0xff),
+                                           r.randint(0x00, 0xff),
+                                           r.randint(0x00, 0xff),
+                                           r.randint(0x00, 0xff),
+                                           r.randint(0x00, 0xff))
+    return mac
+
+
+def run(pair):
+    logging.info("KVM migration running on source host [%s] and destination "
+                 "host [%s]\n", pair[0], pair[1])
+
+    source = hosts.create_host(pair[0])
+    dest = hosts.create_host(pair[1])
+    source_at = autotest.Autotest(source)
+    dest_at = autotest.Autotest(dest)
+
+    cfg_file = os.path.join(KVM_DIR, "tests_base.cfg")
+
+    if not os.path.exists(cfg_file):
+        raise error.JobError("Config file %s was not found", cfg_file)
+
+    # Get test set (dictionary list) from the configuration file
+    cfg = kvm_config.config()
+    test_variants = """
+image_name(_.*)? ?<= /tmp/kvm_autotest_root/images/
+cdrom(_.*)? ?<= /tmp/kvm_autotest_root/
+floppy ?<= /tmp/kvm_autotest_root/
+Linux:
+    unattended_install:
+        kernel ?<= /tmp/kvm_autotest_root/
+        initrd ?<= /tmp/kvm_autotest_root/
+qemu_binary = /usr/libexec/qemu-kvm
+qemu_img_binary = /usr/bin/qemu-img
+only qcow2
+only virtio_net
+only virtio_blk
+only smp2
+only no_pci_assignable
+only smallpages
+only Fedora.13.64
+only migrate_multi_host
+nic_mode = tap
+nic_mac_nic1 = %s
+""" % (generate_mac_address())
+    cfg.fork_and_parse(cfg_file, test_variants)
+    test_dicts = cfg.get_list()
+
+    source_control_file = dest_control_file = """
+kvm_test_dir = os.path.join(os.environ['AUTODIR'],'tests/kvm')
+sys.path.append(kvm_test_dir)\n
+"""
+    for params in test_dicts:
+        params['srchost'] = source.ip
+        params['dsthost'] = dest.ip
+        params['rootdir'] = rootdir
+
+        source_params = params.copy()
+        source_params['role'] = "source"
+
+        dest_params = params.copy()
+        dest_params['role'] = "destination"
+        dest_params['migration_mode'] = "tcp"
+
+        # Report the parameters we've received
+        print "Test parameters:"
+        keys = params.keys()
+        keys.sort()
+        for key in keys:
+            logging.debug("    %s = %s", key, params[key])
+
+        source_control_file += "job.run_test('kvm', tag='%s', params=%s)" % (source_params['shortname'], source_params)
+        dest_control_file += "job.run_test('kvm', tag='%s', params=%s)" % (dest_params['shortname'], dest_params)
+
+        logging.info('Source control file:\n%s', source_control_file)
+        logging.info('Destination control file:\n%s', dest_control_file)
+        dest_command = subcommand(dest_at.run,
+                                  [dest_control_file, dest.hostname])
+
+        source_command = subcommand(source_at.run,
+                                    [source_control_file, source.hostname])
+
+        parallel([dest_command, source_command])
+
+# Grab the pairs (and failures)
+(pairs, failures) = utils.form_ntuples_from_machines(machines, 2)
+
+# Log the failures
+for failure in failures:
+    job.record("FAIL", failure[0], "kvm", failure[1])
+
+# Now run through each pair and run
+job.parallel_simple(run, pairs, log=False)
-- 
1.7.3.4

[PATCH 4/5] KVM test: Modifications on the migrate utility function

[Lucas Meneghel Rodrigues]

In order to accomodate multi-host migration, make migrate
accept 2 additional parameters, dest_host and mig_port.
Also, handle properly the case where we are doing migration
inside the same host.

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/kvm_test_utils.py |   50 +++++++++++++++++++++++------------
 1 files changed, 33 insertions(+), 17 deletions(-)

diff --git a/client/tests/kvm/kvm_test_utils.py b/client/tests/kvm/kvm_test_utils.py
index d0f2e6c..2a3a062 100644
--- a/client/tests/kvm/kvm_test_utils.py
+++ b/client/tests/kvm/kvm_test_utils.py
@@ -134,7 +134,7 @@ def reboot(vm, session, method="shell", sleep_before_reset=10, nic_index=0,
 
 def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp",
             mig_cancel=False, offline=False, stable_check=False,
-            clean=False, save_path=None):
+            clean=False, save_path=None, dest_host='localhost', mig_port=None):
     """
     Migrate a VM locally and re-register it in the environment.
 
@@ -144,7 +144,10 @@ def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp",
     @param mig_timeout: timeout value for migration.
     @param mig_protocol: migration protocol
     @param mig_cancel: Test migrate_cancel or not when protocol is tcp.
-    @return: The post-migration VM.
+    @param dest_host: Destination host (defaults to 'localhost').
+    @param mig_port: Port that will be used for migration.
+    @return: The post-migration VM, in case of same host migration, True in
+            case of multi-host migration.
     """
     def mig_finished():
         o = vm.monitor.info("migrate")
@@ -182,18 +185,25 @@ def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp",
             raise error.TestFail("Timeout expired while waiting for migration "
                                  "to finish")
 
-    dest_vm = vm.clone()
-    if stable_check:
+    if dest_host == 'localhost':
+        dest_vm = vm.clone()
+
+    if (dest_host == 'localhost') and stable_check:
         # Pause the dest vm after creation
         dest_vm.params['extra_params'] = (dest_vm.params.get('extra_params','')
                                           + ' -S')
 
-    if not dest_vm.create(migration_mode=mig_protocol, mac_source=vm):
-        raise error.TestError("Could not create dest VM")
+    if dest_host == 'localhost':
+        if not dest_vm.create(migration_mode=mig_protocol, mac_source=vm):
+            raise error.TestError("Could not create dest VM")
+
     try:
         try:
             if mig_protocol == "tcp":
-                uri = "tcp:localhost:%d" % dest_vm.migration_port
+                if dest_host == 'localhost':
+                    uri = "tcp:localhost:%d" % dest_vm.migration_port
+                else:
+                    uri = 'tcp:%s:%d' % (dest_host, mig_port)
             elif mig_protocol == "unix":
                 uri = "unix:%s" % dest_vm.migration_file
             elif mig_protocol == "exec":
@@ -212,11 +222,12 @@ def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp",
                     raise error.TestFail("Failed to cancel migration")
                 if offline:
                     vm.monitor.cmd("cont")
-                dest_vm.destroy(gracefully=False)
+                if dest_host == 'localhost':
+                    dest_vm.destroy(gracefully=False)
                 return vm
             else:
                 wait_for_migration()
-                if stable_check:
+                if (dest_host == 'localhost') and stable_check:
                     save_path = None or "/tmp"
                     save1 = os.path.join(save_path, "src")
                     save2 = os.path.join(save_path, "dst")
@@ -231,14 +242,15 @@ def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp",
                         raise error.TestFail("Mismatch of VM state before "
                                              "and after migration")
 
-                if offline:
+                if (dest_host == 'localhost') and offline:
                     dest_vm.monitor.cmd("cont")
         except:
-            dest_vm.destroy()
+            if dest_host == 'localhost':
+                dest_vm.destroy()
             raise
 
     finally:
-        if stable_check and clean:
+        if (dest_host == 'localhost') and stable_check and clean:
             logging.debug("Cleaning the state files")
             if os.path.isfile(save1):
                 os.remove(save1)
@@ -253,19 +265,23 @@ def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp",
     else:
         raise error.TestFail("Migration ended with unknown status")
 
-    if "paused" in dest_vm.monitor.info("status"):
-        logging.debug("Destination VM is paused, resuming it...")
-        dest_vm.monitor.cmd("cont")
+    if dest_host == 'localhost':
+        if "paused" in dest_vm.monitor.info("status"):
+            logging.debug("Destination VM is paused, resuming it...")
+            dest_vm.monitor.cmd("cont")
 
     # Kill the source VM
     vm.destroy(gracefully=False)
 
     # Replace the source VM with the new cloned VM
-    if env is not None:
+    if (dest_host == 'localhost') and (env is not None):
         kvm_utils.env_register_vm(env, vm.name, dest_vm)
 
     # Return the new cloned VM
-    return dest_vm
+    if dest_host == 'localhost':
+        return dest_vm
+    else:
+        return vm
 
 
 def stop_windows_service(session, service, timeout=120):
-- 
1.7.3.4

[PATCH 5/5] KVM test: kvm_vm: Allow NIC MACs to be defined on config file

[Lucas Meneghel Rodrigues]

In order to allow the multi-host migration test to start
VMs with the same MAC address, allow nic addresses to
be defined on the config file, such as:

nic_mac_nic1 = AA:BB:CC:DD:EE

This way we can generate a MAC on the server control file
and distribute this to the new VMs that are going to be
created.

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/kvm_vm.py |   14 +++++++++++---
 1 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py
index 416b827..b1aefb6 100755
--- a/client/tests/kvm/kvm_vm.py
+++ b/client/tests/kvm/kvm_vm.py
@@ -598,14 +598,22 @@ class VM:
                 self.uuid = f.read().strip()
                 f.close()
 
+
+
             # Generate or copy MAC addresses for all NICs
             num_nics = len(kvm_utils.get_sub_dict_names(params, "nics"))
             for vlan in range(num_nics):
-                mac = mac_source and mac_source.get_mac_address(vlan)
-                if mac:
+                nic_name = kvm_utils.get_sub_dict_names(params, "nics")[vlan]
+                nic_params = kvm_utils.get_sub_dict(params, nic_name)
+                if nic_params.get("nic_mac", None):
+                    mac = nic_params.get("nic_mac")
                     kvm_utils.set_mac_address(self.instance, vlan, mac)
                 else:
-                    kvm_utils.generate_mac_address(self.instance, vlan)
+                    mac = mac_source and mac_source.get_mac_address(vlan)
+                    if mac:
+                        kvm_utils.set_mac_address(self.instance, vlan, mac)
+                    else:
+                        kvm_utils.generate_mac_address(self.instance, vlan)
 
             # Assign a PCI assignable device
             self.pci_assignable = None
-- 
1.7.3.4