* Re: [Autotest] [PATCH] Add a server-side test - kvm_migration
2009-12-07 10:05 ` [Autotest] [PATCH] Add a server-side test - kvm_migration sudhir kumar
@ 2009-12-08 6:14 ` Yolkfull Chow
0 siblings, 0 replies; 2+ messages in thread
From: Yolkfull Chow @ 2009-12-08 6:14 UTC (permalink / raw)
To: sudhir kumar; +Cc: autotest, skannery, kvm-devel
On Mon, Dec 07, 2009 at 03:35:54PM +0530, sudhir kumar wrote:
> Resending with proper cc list :(
>
> On Mon, Dec 7, 2009 at 2:43 PM, sudhir kumar <smalikphy@gmail.com> wrote:
> > Thanks for initiating the server side implementation of migration. Few
> > comments below
> >
> > On Fri, Dec 4, 2009 at 1:48 PM, Yolkfull Chow <yzhou@redhat.com> wrote:
> >> This patch will add a server-side test namely kvm_migration. Currently,
> >> it will use existing KVM client test framework and add a new file
> >> kvm_migration.py to help judge executing routine: source machine or dest
> >> machine.
> >>
> >> * One thing need to be considered/improved:
> >> Whether we parse the kvm_tests.cfg on server machine or on client machines?
> >> If parse it on client machines, we need to fix one problem that adding
> >> 'start_vm_for_migration' parameter into dict which generated on dest machine.
> > I think we can not manage with client side parsing without adding too
> > much complexity. So let us continue parsing on the server side only
> > for remote migration. Also as the patch does, keep the local migration
> > under the client also. I do not like adding test variants in
> > migration_control.srv. Comments below...
> >>
> >> So far I choose parsing kvm_tests.cfg on server machine, and then add
> >> 'start_vm_for_migration' into dict cloned from original test dict for dest
> >> machine.
> >>
> >> * In order to run this test so far, we need to setup NFS for both
> >> source and dest machines.
> >>
> >> Signed-off-by: Yolkfull Chow <yzhou@redhat.com>
> >> ---
> >> client/tests/kvm/kvm_migration.py | 165 ++++++++++++++++++++++++++++++++
> >> client/tests/kvm/kvm_test_utils.py | 27 +++---
> >> client/tests/kvm/kvm_tests.cfg.sample | 2 +
> >> client/tests/kvm_migration | 1 +
> >> server/tests/kvm/migration_control.srv | 137 ++++++++++++++++++++++++++
> >> 5 files changed, 320 insertions(+), 12 deletions(-)
> >> create mode 100644 client/tests/kvm/kvm_migration.py
> >> create mode 120000 client/tests/kvm_migration
> >> create mode 100644 server/tests/kvm/migration_control.srv
> >>
> >> diff --git a/client/tests/kvm/kvm_migration.py b/client/tests/kvm/kvm_migration.py
> >> new file mode 100644
> >> index 0000000..52cd3cd
> >> --- /dev/null
> >> +++ b/client/tests/kvm/kvm_migration.py
> >> @@ -0,0 +1,165 @@
> >> +import sys, os, time, logging, commands, socket
> >> +from autotest_lib.client.bin import test
> >> +from autotest_lib.client.common_lib import error
> >> +import kvm_utils, kvm_preprocessing, common, kvm_vm, kvm_test_utils
> >> +
> >> +
> >> +class kvm_migration(test.test):
> >> + """
> >> + KVM migration test.
> >> +
> >> + @copyright: Red Hat 2008-2009
> >> + @see: http://www.linux-kvm.org/page/KVM-Autotest/Client_Install
> >> + (Online doc - Getting started with KVM testing)
> >> +
> >> + Migration execution progress:
> >> +
> >> + source host dest host
> >> + ------------------------------------------------------------------
> >> + log into guest
> >> + ------------------------------------------------------------------
> >> + start socket server
> >> +
> >> + ---- wait 30 secs -------------- wait login_timeout+30 secs-------
> >> +
> >> + accept connection connect to socket server,send mig_port
> >> + ------------------------------------------------------------------
> >> + start migration
> >> +
> >> + ---- wait 30 secs -------------- wait mig_timeout+30 secs---------
> >> +
> >> + try to log into migrated guest
> >> + ------------------------------------------------------------------
> >> +
> >> + """
> >> + version = 1
> >> + def initialize(self):
> >> + pass
> >> +
> >> +
> >> + def run_once(self, params):
> >> + """
> >> + Setup remote machine and then execute migration.
> >> + """
> >> + # Check whether remote machine is ready
> >> + dsthost = params.get("dsthost")
> >> + srchost = params.get("srchost")
> >> + image_path = os.path.join(self.bindir, "images")
> >> +
> >> + rootdir = params.get("rootdir")
> >> + iso = os.path.join(rootdir, 'iso')
> >> + images = os.path.join(rootdir, 'images')
> >> + qemu = os.path.join(rootdir, 'qemu')
> >> + qemu_img = os.path.join(rootdir, 'qemu-img')
> >> +
> >> + def link_if_not_exist(ldir, target, link_name):
> >> + t = target
> >> + l = os.path.join(ldir, link_name)
> >> + if not os.path.exists(l):
> >> + os.symlink(t,l)
> >> + link_if_not_exist(self.bindir, '../../', 'autotest')
> >> + link_if_not_exist(self.bindir, iso, 'isos')
> >> + link_if_not_exist(self.bindir, images, 'images')
> >> + link_if_not_exist(self.bindir, qemu, 'qemu')
> >> + link_if_not_exist(self.bindir, qemu_img, 'qemu-img')
> >> +
> >> + # Report the parameters we've received and write them as keyvals
> >> + logging.debug("Test parameters:")
> >> + keys = params.keys()
> >> + keys.sort()
> >> + for key in keys:
> >> + logging.debug(" %s = %s", key, params[key])
> >> + self.write_test_keyval({key: params[key]})
> >> +
> >> + # Open the environment file
> >> + env_filename = os.path.join(self.bindir, params.get("env", "env"))
> >> + env = kvm_utils.load_env(env_filename, {})
> >> + logging.debug("Contents of environment: %s" % str(env))
> >> +
> >> + # Preprocess
> >> + kvm_preprocessing.preprocess(self, params, env)
> >> + kvm_utils.dump_env(env, env_filename)
> >> +
> >> + try:
> >> + try:
> >> + # Get the living VM
> >> + vm = kvm_test_utils.get_living_vm(env, params.get("main_vm"))
> >> + migration_test_command = params.get("migration_test_command")
> >> + login_timeout = int(params.get("login_timeout"))
> >> + mig_timeout = int(params.get("mig_timeout"))
> >> + source_addr = (srchost, 50006)
> >> + all = [srchost, dsthost]
> >> +
> >> + # Check whether migration is supported
> >> + s, o = vm.send_monitor_cmd("help info")
> >> + if not "info migrate" in o:
> >> + raise error.TestError("Migration is not supported")
> >> +
> >> + if params.get("role") == "source":
> >> +
> >> + session = kvm_test_utils.wait_for_login(vm,
> >> + timeout=login_timeout)
> >> + ref_output = session.get_command_output(migration_test_command)
> >> +
> >> +
> >> + # Listen on a port to get the migration port received from
> >> + # dest machine
> >> + s = socket.socket()
> >> + s.bind(source_addr)
> >> + s.listen(1)
> >> +
> >> + # Wait 30 seconds for source and dest to reach this point
> >> + self.job.barrier(srchost,'socket_started',30).rendezvous(*all)
> >> +
> >> + conn, addr = s.accept()
> >> + mig_port = int(conn.recv(6))
> >> + s.close()
> >> +
> >> + logging.info("Start migrating now...")
> >> + kvm_test_utils.migrate(vm, dsthost, mig_port, env)
> >> +
> >> + # Wait up to 30 seconds for dest to reach this point
> >> + self.job.barrier(srchost,'mig_finished',30).rendezvous(*all)
> >> +
> >> + session.close()
> >> +
> >> + elif params.get("role") == "dest":
> >> +
> >> + # Wait up to login_timeout+30 seconds for the source to
> >> + # reach this point
> >> + self.job.barrier(dsthost, 'socket_started',
> >> + login_timeout+30).rendezvous(*all)
> >> +
> >> + s = socket.socket()
> >> + s.connect(source_addr)
> >> + s.send("%d" % vm.migration_port)
> >> + s.close()
> >> +
> >> + # Wait up to mig_timeout+30 seconds for the source to
> >> + # reach this point: migration finished
> >> + self.job.barrier(dsthost, 'mig_finished',
> >> + mig_timeout+30).rendezvous(*all)
> >> +
> >> + try:
> >> + session = kvm_test_utils.wait_for_login(vm)
> >> + except:
> >> + raise error.TestFail("Could not log into migrated guest")
> >> +
> >> + if session.get_command_status(migration_test_command) != 0:
> >> + raise error.TestFail("migration_test_command failed!")
> > I think we should compare this output with the contents of ref_output
> > from the source vm ?
Yes, it had been added in my latest improvement.
> >> +
> >> + else:
> >> + raise error.TestError('Invalid role specified')
> >> +
> >> + except Exception, e:
> >> + logging.error("Test failed: %s", e)
> >> + logging.debug("Postprocessing on error...")
> >> + kvm_preprocessing.postprocess_on_error(self, params, env)
> >> + kvm_utils.dump_env(env, env_filename)
> >> + raise
> >> +
> >> + finally:
> >> + # Postprocess
> >> + kvm_preprocessing.postprocess(self, params, env)
> >> + logging.debug("Contents of environment: %s", str(env))
> >> + kvm_utils.dump_env(env, env_filename)
> >> diff --git a/client/tests/kvm/kvm_test_utils.py b/client/tests/kvm/kvm_test_utils.py
> >> index 02ec0cf..3e3bd49 100644
> >> --- a/client/tests/kvm/kvm_test_utils.py
> >> +++ b/client/tests/kvm/kvm_test_utils.py
> >> @@ -106,7 +106,7 @@ def reboot(vm, session, method="shell", sleep_before_reset=10, nic_index=0,
> >> return session
> >>
> >>
> >> -def migrate(vm, env=None):
> >> +def migrate(vm, rem_host, rem_port, env=None, mig_timeout=90):
> > can we use mig_port instead of rem_port or rem_mig_port ?
> >> """
> >> Migrate a VM locally and re-register it in the environment.
> >>
> >> @@ -133,13 +133,14 @@ def migrate(vm, env=None):
> >> if not "info migrate" in o:
> >> raise error.TestError("Migration is not supported")
> >>
> >> - # Clone the source VM and ask the clone to wait for incoming migration
> >> - dest_vm = vm.clone()
> >> - dest_vm.create(for_migration=True)
> >> + if rem_host == "localhost":
> >> + # Clone the source VM and ask the clone to wait for incoming migration
> >> + dest_vm = vm.clone()
> >> + dest_vm.create(for_migration=True)
> >>
> >> try:
> >> # Define the migration command
> >> - cmd = "migrate -d tcp:localhost:%d" % dest_vm.migration_port
> >> + cmd = "migrate -d tcp:%s:%d" % (rem_host, rem_port)
> >> logging.debug("Migrating with command: %s" % cmd)
> >>
> >> # Migrate
> >> @@ -150,7 +151,7 @@ def migrate(vm, env=None):
> >> raise error.TestFail("Migration command failed")
> >>
> >> # Wait for migration to finish
> >> - if not kvm_utils.wait_for(mig_finished, 90, 2, 2,
> >> + if not kvm_utils.wait_for(mig_finished, mig_timeout, 2, 2,
> >> "Waiting for migration to finish..."):
> >> raise error.TestFail("Timeout elapsed while waiting for migration "
> >> "to finish")
> >> @@ -166,15 +167,17 @@ def migrate(vm, env=None):
> >> # Kill the source VM
> >> vm.destroy(gracefully=False)
> >>
> >> - # Replace the source VM with the new cloned VM
> >> - if env is not None:
> >> - kvm_utils.env_register_vm(env, vm.name, dest_vm)
> >> + if rem_host == "localhost":
> >> + # Replace the source VM with the new cloned VM
> >> + if env is not None:
> >> + kvm_utils.env_register_vm(env, vm.name, dest_vm)
> >>
> >> - # Return the new cloned VM
> >> - return dest_vm
> >> + # Return the new cloned VM
> >> + return dest_vm
> >>
> >> except:
> >> - dest_vm.destroy()
> >> + if rem_host == "localhost":
> >> + dest_vm.destroy()
> >> raise
> >>
> >>
> >> diff --git a/client/tests/kvm/kvm_tests.cfg.sample b/client/tests/kvm/kvm_tests.cfg.sample
> >> index 9fb1ba8..788b870 100644
> >> --- a/client/tests/kvm/kvm_tests.cfg.sample
> >> +++ b/client/tests/kvm/kvm_tests.cfg.sample
> >> @@ -81,6 +81,8 @@ variants:
> >> migration_bg_command = "cd /tmp; nohup tcpdump -q -t ip host localhost"
> >> migration_bg_check_command = pgrep tcpdump
> >> migration_bg_kill_command = pkill tcpdump
> >> + mig_timeout = 90
> >> + login_timeout = 240
> >> kill_vm_on_error = yes
> >> iterations = 2
> >> used_mem = 1024
> >> diff --git a/client/tests/kvm_migration b/client/tests/kvm_migration
> >> new file mode 120000
> >> index 0000000..9186877
> >> --- /dev/null
> >> +++ b/client/tests/kvm_migration
> >> @@ -0,0 +1 @@
> >> +kvm
> >> \ No newline at end of file
> >> diff --git a/server/tests/kvm/migration_control.srv b/server/tests/kvm/migration_control.srv
> >> new file mode 100644
> >> index 0000000..9774c43
> >> --- /dev/null
> >> +++ b/server/tests/kvm/migration_control.srv
> >> @@ -0,0 +1,137 @@
> >> +AUTHOR = "Yolkfull Chow <yzhou@redhat.com>"
> >> +TIME = "SHORT"
> >> +NAME = "Migration across Multi-machine"
> >> +TEST_CATEGORY = "Functional"
> >> +TEST_CLASS = 'Virtualization'
> >> +TEST_TYPE = "Server"
> >> +DOC = """
> >> +Migrate KVM guest between two hosts.
> >> +
> >> +Arguments to run_test:
> >> +
> >> +@dict - a dictionary containing all parameters that migration need.
> >> +"""
> >> +
> >> +import sys, os, commands, glob, shutil
> >> +from autotest_lib.server import utils
> >> +
> >> +AUTOTEST_DIR = '/root/devel/upstream/server-migration'
> > ??
> >> +
> >> +CLIENT_TEST_DIR = os.path.join(AUTOTEST_DIR, 'client/tests/')
> >> +KVM_DIR = os.path.join(CLIENT_TEST_DIR, 'kvm')
> >> +
> >> +sample_files = glob.glob(os.path.join(KVM_DIR, '*.cfg.sample'))
> >> +for sample_file in sample_files:
> >> + sample_file_name = os.path.basename(sample_file)
> >> + cfg_file_name = ".".join(sample_file_name.split(".")[:-1])
> >> + cfg_file_path = os.path.join(KVM_DIR, cfg_file_name)
> >> + sample_file_path = os.path.join(KVM_DIR, sample_file)
> >> + shutil.copyfile(sample_file_path, cfg_file_path)
> > We are writing the .cfg.sample files to .cfg files here. I feel the
> > user is supposed to do that as there may be needed some changes(like
> > allowing RHEL to run all the tests etc)
I don't think user should/need log into each client machine and do so many
duplicated works once before running a server-side test.
> >> +
> >> +sys.path.append(KVM_DIR)
> >> +import common, kvm_config
> >> +
> >> +rootdir = '/tmp/kvm_autotest_root'
> >> +
> >> +def run(pair):
> >> + print "KVM migration running on srchost [%s] and desthost [%s]\n" % (
> >> + pair[0], pair[1])
> >> +
> >> + # Edit this to limit the migration that you want
> >> + test_variants = """
> >> +only full
> >> +only qcow2
> >> +only ide
> >> +only default
> >> +only up
> >> +only Fedora.11.32
> >> +no install setup
> >> +no hugepages
> >> +only migrate
> >> +only rtl8139
> >> +"""
> > Somehow this idea does not look to be the best. We should not write
> > test configuration(i.e. variants) at multiple places. May be we can
> > change the migration into two variants: local_migration and
> > remote_migration. Client can run the local_migration but should
> > exclude remote_migration. On the other side server should be able to
> > run both of them.
> > (NB: I do not have any hands on execution experience with autotest
> > server, all my comments are based on my understanding built by code
> > browsing)
As metioned above, what user could touch (in WebUI) before actually running
a server-side test is the server control file. As a result of that,
we can only limit the test via this method AFAIK.
Please give the furthur implementation suggestion if you have a better idea.
Thanks,
> >> +
> >> + source = hosts.create_host(pair[0])
> >> + dest = hosts.create_host(pair[1])
> >> +
> >> + source_at = autotest.Autotest(source)
> >> + source_at.install(source)
> >> + dest_at = autotest.Autotest(dest)
> >> + dest_at.install(dest)
> >> +
> >> + # FIXME: should parse config file on client machines?
> >> + filename = os.path.join(KVM_DIR, "kvm_tests.cfg.sample")
> >> + sample_f = open(filename, 'r')
> >> + new_contents = sample_f.readlines()[:-1]
> >> + sample_f.close()
> >> +
> >> + new_contents = "".join(new_contents)
> > We are copying contents of .cfg.sample file while at this stage both
> > .cfg and .cfg.sample are same ?
Of course not. The last line which is used to control final test is commented
and therefore string that user can edit in server control file
is added to kvm_tests.cfg. In this way, user could limit the test.
> >> + new_contents += test_variants
> >> +
> >> + filename = os.path.join(KVM_DIR, 'kvm_tests.cfg')
> >> + f = open(filename, 'w')
> >> + f.write(new_contents)
> >> + f.close()
> >> +
> >> + if not os.path.exists(filename):
> >> + print "[ERROR] Test config file doesn't found"
> >> + sys.exit(1)
> >> +
> >> + # Get test set (dictionary list) from the configuration file
> >> + cfg = kvm_config.config(filename)
> >> +
> >> + filename = os.path.join(KVM_DIR, "kvm_address_pools.cfg")
> >> + if os.path.exists(filename):
> >> + cfg.parse_file(filename)
> >> + hostname = os.uname()[1].split(".")[0]
> >> + if cfg.filter("^" + hostname):
> >> + cfg.parse_string("only ^%s" % hostname)
> >> + else:
> >> + cfg.parse_string("only ^default_host")
> >> +
> >> + list = cfg.get_list()
> >> +
> >> + # Control file template for client machine
> >> + control_string = "job.run_test('kvm_migration', params=%s)"
> >> +
> >> + for vm_dict in list:
> >> +
> >> + vm_dict['srchost'] = source.ip
> >> + vm_dict['dsthost'] = dest.ip
> >> + vm_dict['display'] = 'vnc'
> >> + vm_dict['rootdir'] = rootdir
> >> +
> >> + source_dict = vm_dict.copy()
> >> + dest_dict = vm_dict.copy()
> >> +
> >> + source_dict['role'] = "source"
> >> +
> >> + dest_dict['role'] = "dest"
> >> + dest_dict['start_vm_for_migration'] = "yes"
> >> +
> >> + # Report the parameters we've received
> >> + print "Test parameters:"
> >> + keys = vm_dict.keys()
> >> + keys.sort()
> >> + for key in keys:
> >> + print " " + str(key) + " = " + str(vm_dict[key])
> >> +
> >> + source_control_file = ''.join([control_string % source_dict])
> >> + dest_control_file = ''.join([control_string % dest_dict])
> >> +
> >> + dest_command = subcommand(dest_at.run,
> >> + [dest_control_file, dest.hostname])
> >> + source_command = subcommand(source_at.run,
> >> + [source_control_file, source.hostname])
> >> +
> >> + parallel([dest_command, source_command])
> >> +
> >> +# Grab the pairs (and failures)
> >> +(pairs, failures) = utils.form_ntuples_from_machines(machines, 2)
> >> +
> >> +# Log the failures
> >> +for failure in failures:
> >> + job.record("FAIL", failure[0], "kvm", failure[1])
> >> +
> >> +# Now run through each pair and run
> >> +job.parallel_simple(run, pairs, log=False)
> >> --
> >> 1.6.5.2
> >>
> >> _______________________________________________
> >> Autotest mailing list
> >> Autotest@test.kernel.org
> >> http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
> >>
> >
> > Rest of the patch looks to be clean to me.
Thanks for review. :)
Yolkfull Chow
> >
> > --
> > Sudhir Kumar
> >
>
>
>
> --
> Sudhir Kumar
^ permalink raw reply [flat|nested] 2+ messages in thread