About implementation of KVM server-side migration in autotest

[Yolkfull Chow <yzhou@redhat.com>]

Hi Michael and Lucas,

Would you please give me some comments/suggestions based on
my initial design of it? Thanks in advance. :)


============ KVM server-side migration design =================

1 for generating migration test dicts, two methods:

1.1 use existent framework of kvm client test
1.2 new a script named kvm_migration.py under dir kvm, initial version could be:

kvm_migration.py
--------
import sys, os, time, logging, commands
from autotest_lib.client.bin import test
from autotest_lib.client.common_lib import error
import kvm_utils, kvm_preprocessing, common, kvm_vm, kvm_test_utils


class kvm_migration(test.test):
    """
    KVM migration test.

    @copyright: Red Hat 2008-2009
    @see: http://www.linux-kvm.org/page/KVM-Autotest/Client_Install
            (Online doc - Getting started with KVM testing)
    """
    version = 1
    def initialize(self):
        pass


    def setup(self):
        """
        Setup environment like NFS mount etc.
        """
        pass


    def run_once(self, params):
        """
        Setup remote machine and then execute migration.
        """
        # Check whether remote machine is ready
        dsthost = params.get("dsthost")
        srchost = params.get("srchost")
        image_path = os.path.join(self.bindir, "images")

        rootdir = params.get("rootdir")
        iso = os.path.join(rootdir, 'iso')
        images = os.path.join(rootdir, 'images')
        qemu = os.path.join(rootdir, 'qemu')
        qemu_img = os.path.join(rootdir, 'qemu-img')

        def link_if_not_exist(ldir, target, link_name):
            t = target
            l = os.path.join(ldir, link_name)
            if not os.path.exists(l):
                os.symlink(t,l)
        link_if_not_exist(self.bindir, '../../', 'autotest')
        link_if_not_exist(self.bindir, iso, 'isos')
        link_if_not_exist(self.bindir, images, 'images')
        link_if_not_exist(self.bindir, qemu, 'qemu')
        link_if_not_exist(self.bindir, qemu_img, 'qemu-img')

        try:
            image_real_path = os.readlink(image_path)
        except OSError:
            raise error.TestError("Readlink of image dir failed")

        def setup_dest(srchost, path):
            """
            Mount NFS directory from source host.
            """
            cmd = "mount |grep -q %s" % srchost
            if os.system(cmd):
                mnt_cmd = "mount %s:%s %s" % (srchost,
                                              path,
                                              path)
                s, o = commands.getstatusoutput(mnt_cmd)
                if s != 0:
                    raise error.TestError("Mount srchost failed: %s" % o)
        
        def setup_source(path):
            """
            Setup NFS mount point.
            """
            export_string = "%s *(rw,no_root_squash)" % path
            export_file = '/etc/exports'
            f = open(export_file)
            if not export_string in f.read().strip():
                try:
                    f.write(export_string)
                except IOError:
                    raise error.TestError("Failed to write to exports file")

                cmd = "service nfs restart && exportfs -a"
                if os.system(cmd):
                    raise error.TestError("Failed to restart NFS on source")

        if params.get("role") == "dest":
            setup_dest(srchost, image_real_path)
        elif params.get("role") == "source":
            setup_source(image_real_path)

        # Report the parameters we've received and write them as keyvals
        logging.debug("Test parameters:")
        keys = params.keys()
        keys.sort()
        for key in keys:
            logging.debug("    %s = %s", key, params[key])
            self.write_test_keyval({key: params[key]})

        # Open the environment file
        env_filename = os.path.join(self.bindir, params.get("env", "env"))
        env = kvm_utils.load_env(env_filename, {})
        logging.debug("Contents of environment: %s" % str(env))


        # Preprocess
        kvm_preprocessing.preprocess(self, params, env)
        kvm_utils.dump_env(env, env_filename)

        try:
            try:
                # Get the living VM
                vm = kvm_test_utils.get_living_vm(env, params.get("main_vm"))

                if params.get("role") == "source":
                    s, o = vm.send_monitor_cmd("help info")
                    if not "info migrate" in o:
                        raise error.TestError("Migration is not supported")

                    session = kvm_test_utils.wait_for_login(vm)
                    migration_test_command = params.get("migration_test_command")
                    reference_output = session.get_command_output(migration_test_command)

                    kvm_test_utils.migrate(vm, dsthost, vm.migration_port, env)
                    session.close()
                elif params.get("role") == "dest":
                    try:
                        mig_timeout = int(params.get("mig_timeout"))
                        session = kvm_test_utils.wait_for_login(vm,
                                                           timeout=mig_timeout)
                    except:
                        raise error.TestFail("Could not log into migrated guest")

            except Exception, e:
                logging.error("Test failed: %s", e)
                logging.debug("Postprocessing on error...")
                kvm_preprocessing.postprocess_on_error(self, params, env)
                kvm_utils.dump_env(env, env_filename)
                raise

        finally:
            # Postprocess
            kvm_preprocessing.postprocess(self, params, env)
            logging.debug("Contents of environment: %s", str(env))
            kvm_utils.dump_env(env, env_filename)
--------  

   But there will be a problem:  where can we edit the config file
   to generate differnt dicts for migration test? Hard code the parameters in control.srv? 

   It will be easier to resolve the problem of making up kvm_tests.cfg for both client
   machines. 

2 just pass 'start_vm_for_migration = yes' to dest host

3 source host will wait until dest host telling it's ready using socket
  communication (any existent implement/function in autotest framework? )

4 and then change the role of 'source machine' and 'dest machine' to implement ping-pong
  migrate.

Initial version of control.srv for server-migration:

-----
AUTHOR = "Yolkfull Chow <yzhou@redhat.com>"
TIME = "SHORT"
NAME = "Migration across Multi-machine"
TEST_CATEGORY = "Functional"
TEST_CLASS = 'Virtualization'
TEST_TYPE = "Server"
DOC = """
Migrate KVM guest between two hosts.

Arguments to run_test:

@dict - a dictionary containing all parameters that migration need.
"""

import sys, os, commands 
from autotest_lib.server import utils

KVM_DIR = os.path.join('/root/devel/upstream/server-mig', 'client/tests/kvm')
sys.path.insert(0, KVM_DIR)
import kvm_config

rootdir = '/tmp/kvm_autotest_root'

def run(pair):
    print "KVM migration running on srchost [%s] and desthost [%s]\n" % (
                                                        pair[0], pair[1])

    source = hosts.create_host(pair[0])
    dest = hosts.create_host(pair[1])

    source_at = autotest.Autotest(source)
    source_at.install(source)
    dest_at = autotest.Autotest(dest)
    dest_at.install(dest)

    # ----------------------------------------------------------
    # Get test set (dictionary list) from the configuration file
    # ----------------------------------------------------------
    filename = os.path.join(KVM_DIR, "kvm_tests.cfg")
    cfg = kvm_config.config(filename)
    
    # Make only dictionaries that migration needs
    cfg.parse_string("only migrate")
    
    filename = os.path.join(KVM_DIR, "kvm_address_pools.cfg")
    if os.path.exists(filename):
        cfg.parse_file(filename)
        hostname = os.uname()[1].split(".")[0]
        if cfg.filter("^" + hostname):
            cfg.parse_string("only ^%s" % hostname)
        else:
            cfg.parse_string("only ^default_host")
    list = cfg.get_list()


    # Control file template for client machine
    control_string = "job.run_test('kvm_migration', params=%s)"

    for vm_dict in list:

        vm_dict['srchost'] = source.ip
        vm_dict['dsthost'] = dest.ip
        vm_dict['display'] = 'vnc'
        vm_dict['rootdir'] = rootdir

        source_dict = vm_dict.copy()
        dest_dict = vm_dict.copy()

        source_dict['role'] = "source"

        dest_dict['role'] = "dest"
        dest_dict['start_vm_for_migration'] = "yes"

        # Report the parameters we've received
        print "Test parameters:"
        keys = vm_dict.keys()
        keys.sort()
        for key in keys:
            print "    " + str(key) + " = " + str(vm_dict[key])

        source_control_file = ''.join([control_string % source_dict])
        dest_control_file = ''.join([control_string % dest_dict])

        dest_command = subcommand(dest_at.run,
                                    [dest_control_file, dest.hostname])
        source_command = subcommand(source_at.run,
                                    [source_control_file, source.hostname])

        parallel([dest_command, source_command])

# grab the pairs (and failures)
(pairs, failures) = utils.form_ntuples_from_machines(machines, 2)

# log the failures
for failure in failures:
    job.record("FAIL", failure[0], "kvm", failure[1])

# now run through each pair and run
job.parallel_simple(run, pairs, log=False)
--------