From mboxrd@z Thu Jan 1 00:00:00 1970 From: =?ISO-8859-2?Q?Luk=E1=B9_Doktor?= Subject: Re: [KVM-AUTOTEST PATCH 2/2] Add KSM test Date: Mon, 31 Aug 2009 11:48:07 +0200 Message-ID: <4A9B9C57.2020408@redhat.com> References: <4A9B97E5.3000109@redhat.com> Mime-Version: 1.0 Content-Type: text/plain; charset=ISO-8859-2; format=flowed Content-Transfer-Encoding: QUOTED-PRINTABLE To: KVM list , Autotest mailing list Return-path: Received: from mx1.redhat.com ([209.132.183.28]:15236 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1751095AbZHaJsI (ORCPT ); Mon, 31 Aug 2009 05:48:08 -0400 In-Reply-To: <4A9B97E5.3000109@redhat.com> Sender: kvm-owner@vger.kernel.org List-ID: This is an actual KSM test. It allows to test merging resp splitting the pages in serial, parallel=20 or both. Also you can specify an overcommit ratio for KSM overcommit=20 testing. We were forced to destroy all previous defined vms and to create them=20 inside the test (similar to stress_boot), because we don't know how man= y=20 machines will be required during the vm preparation. Second nasty thing is filling the memory by the guests. We didn't find=20 better way to test filled memory without the python(kvm-autotest) fall.= =20 This version continue filling until a small reserve than destroy=20 previous machines and let the actual machine finish the work. Signed-off-by: Luk=E1=B9 Doktor Signed-off-by: Ji=F8=ED =AEupka --- client/tests/kvm/kvm.py | 2 + client/tests/kvm/kvm_tests.cfg.sample | 17 + client/tests/kvm/kvm_tests.py | 548=20 +++++++++++++++++++++++++++++++++ 3 files changed, 567 insertions(+), 0 deletions(-) diff --git a/client/tests/kvm/kvm.py b/client/tests/kvm/kvm.py index 4930e80..b9839df 100644 --- a/client/tests/kvm/kvm.py +++ b/client/tests/kvm/kvm.py @@ -53,6 +53,8 @@ class kvm(test.test): "yum_update": test_routine("kvm_tests",=20 "run_yum_update"), "autotest": test_routine("kvm_tests", "run_autote= st"), "kvm_install": test_routine("kvm_install",=20 "run_kvm_install"), + "ksm": + test_routine("kvm_tests", "run_ksm"), "linux_s3": test_routine("kvm_tests", "run_linux_= s3"), "stress_boot": test_routine("kvm_tests",=20 "run_stress_boot"), "timedrift": test_routine("kvm_tests",=20 "run_timedrift"), diff --git a/client/tests/kvm/kvm_tests.cfg.sample=20 b/client/tests/kvm/kvm_tests.cfg.sample index a83ef9b..f4a41b9 100644 --- a/client/tests/kvm/kvm_tests.cfg.sample +++ b/client/tests/kvm/kvm_tests.cfg.sample @@ -100,6 +100,23 @@ variants: test_name =3D disktest test_control_file =3D disktest.control + - ksm: + # Don't preprocess any vms as we need to change it's params + vms =3D '' + image_snapshot =3D yes + kill_vm_gracefully =3D no + type =3D ksm + variants: + - ratio_3: + ksm_ratio =3D 3 + - ratio_10: + ksm_ratio =3D 10 + variants: + - serial + ksm_test_size =3D "serial" + - paralel + ksm_test_size =3D "paralel" + - linux_s3: install setup type =3D linux_s3 diff --git a/client/tests/kvm/kvm_tests.py b/client/tests/kvm/kvm_tests= =2Epy index b100269..ada4c6b 100644 --- a/client/tests/kvm/kvm_tests.py +++ b/client/tests/kvm/kvm_tests.py @@ -462,6 +462,554 @@ def run_yum_update(test, params, env): session.close() +def run_ksm(test, params, env): + """ + Test how KSM (Kernel Shared Memory) act with more than physical=20 memory is + used. In second part is also tested, how KVM can handle the situat= ion, + when the host runs out of memory (expected is to pause the guest=20 system, + wait until some process returns the memory and bring the guest bac= k=20 to life) + + @param test: kvm test object. + @param params: Dictionary with test parameters. + @param env: Dictionary with the test wnvironment. + """ + # We are going to create the main VM so we use kvm_preprocess func= tions + # FIXME: not a nice thing + import kvm_preprocessing + import random + import socket + import select + import math + + class allocator_com: + """ + This class is used for communication with the allocator + """ + def __init__(self, vm, _port, _host=3D'127.0.0.1'): + self.vm =3D vm + self.PORT =3D _port + self.HOST =3D _host + self.socket =3D socket.socket(socket.AF_INET, socket.SOCK_= STREAM) + self.isConnect =3D False + + def __str__(self): + return self.vm + ":" + self.HOST + ":" + str(self.PORT) + + def connect(self): + print self + logging.debug("ALLOC: connect to %s", self.vm) + try: + self.socket.connect((self.HOST, self.PORT)) + except: + raise error.TestFail("ALLOC: Could not establish the "= \ + "communication with %s" % (self.v= m)) + self.isConnect =3D True + + def isConnected(self): + return self.isConnect; + + def readsize(self): + read,write,error =3D=20 select.select([self.socket.fileno()],[],[],0.5) + size =3D 0 + if (self.socket.fileno() in read): + data =3D self.socket.recv(1); + size =3D ""; + while data[0] !=3D ':': + size =3D size + data[0] + data =3D self.socket.recv(1) + return int(size) + + def _recv(self): + msg =3D "" + read, write, error =3D select.select([self.socket.fileno()= ],\ + [], [], 0.5) + if (self.socket.fileno() in read): + size =3D self.readsize() + msg =3D self.socket.recv(size) + if (len(msg) < size): + raise error.TestFail("ALLOC: Could not recive the=20 message") + + logging.debug("ALLOC: output '%s' from %s" % (msg, self.vm= )) + return msg + + def recv(self, wait=3D1, loops=3D20): + out =3D "" + log =3D "" + while not out.startswith("PASS") and not=20 out.startswith("FAIL"): + logging.debug("Sleep(%d)" % (wait)) + time.sleep(wait) + log +=3D out + out =3D self._recv() + + if loops =3D=3D 0: + logging.error(repr(out)) + raise error.TestFail("Command wasn't finished unti= l=20 DL") + loops =3D loops - 1 + + if not out.startswith("PASS"): + logging.error("Allocator failed on guest %s\nAttaching= =20 the"\ + "recent log" % (self.vm)) + raise error.TestFail(log) + + return out + + + def send(self, command, data=3D""): + msg =3D str(len(command) + len(data) + 3) + msg +=3D ":" + command + ":" + data + ";" + logging.debug("ALLOC: execute %s on %s" %(repr(msg), self.= vm)) + try: + self.socket.sendall(msg) + except: + raise error.TestFail("ALLOC: Could not send the messag= e") + + def disconnect(self): + logging.debug("ALLOC: disconnect") + self.send("exit") + self.recv() + time.sleep(5) + self.socket.close() + self.isConnect =3D False + + def get_stat(lvms): + """ + Get statistics in format: + Host: memfree =3D XXXM; Guests memsh =3D {XXX,XXX,...} + + @params lvms: List of VMs + """ + if not isinstance(lvms, list): + raise error.TestError("get_stat: parameter have to be=20 proper list") + + try: + stat =3D "Host: memfree =3D " + stat +=3D str(int(os.popen("cat /proc/meminfo | grep MemFr= ee")\ + .readline().split()[1]) / 1024) += =20 "M; " + stat +=3D "swapfree =3D " + stat +=3D str(int(os.popen("cat /proc/meminfo | grep SwapF= ree")\ + .readline().split()[1]) / 1024) += =20 "M; " + except: + raise error.TestFail("Could not fetch free memory info") + + + stat +=3D "Guests memsh =3D {" + for vm in lvms: + try: + cmd =3D "cat /proc/%d/statm" % vm.pid + shm =3D int(os.popen(cmd).readline().split()[2]) + # statm stores informations in pages, recalculate to M= B + shm =3D shm * 4 / 1024 + stat +=3D "%dM; " % (shm) + except: + raise error.TestError("Could not fetch shmem info from= =20 proc") + stat =3D stat[0:-2] + "}" + return stat + + + + + + logging.info("Starting phase 0: Initialization") + # host_reserve: mem reserve keept for the host system to run + host_reserve =3D 256 + # guest_reserve: mem reserve which is not used by allocator on the= =20 guests + guest_reserve =3D 256 + max_alloc =3D 10 + max_vms =3D params.get("max_vms") + if max_vms: + max_vms =3D int(max_vms) + else: + max_vms =3D 2 + overcommit =3D params.get("ksm_overcommit_ratio") + if overcommit: + overcommit =3D float(overcommit) + else: + overcommit =3D 2.0 + # vmsc: count of all used VMs + vmsc =3D int(overcommit) + 1 + vmsc =3D max(vmsc, max_vms) + + if (params['ksm_test_size'] =3D=3D "paralel") : + host_mem =3D (int(os.popen("grep MemTotal: /proc/meminfo")\ + .readline().split()[1]) / 1024 - host_reserve) + vmsc =3D 1 + overcommit =3D 1 + mem =3D host_mem + # 32bit system adjustment + if not params['image_name'].endswith("64"): + logging.debug("Probably i386 guest architecture, "\ + "max allocator mem =3D 2G") + # Guest can have more than 2G but kvm mem + 1MB (allocator= =20 itself) can't + if (host_mem > 2048): + mem =3D 2047 + + + if os.popen("uname -i").readline().startswith("i386"): + logging.debug("Host is i386 architecture, max guest mem is= 2G") + # Guest system with qemu overhead (64M) can't have more th= an 2G + if mem > 2048 - 64: + mem =3D 2048 - 64 + + else: + host_mem =3D (int(os.popen("grep MemTotal: /proc/meminfo")\ + .readline().split()[1]) / 1024 - host_reserve) + # mem: Memory of the guest systems. Maximum must be less than=20 amount of the + # host's physical ram + mem =3D int(overcommit * host_mem / vmsc) + + # 32bit system adjustment + if not params['image_name'].endswith("64"): + logging.debug("Probably i386 guest architecture, "\ + "max allocator mem =3D 2G") + # Guest can have more than 2G but kvm mem + 1MB (allocator= =20 itself) can't + if mem-guest_reserve-1 > 2048: + vmsc =3D=20 int(math.ceil((host_mem*overcommit)/(2048.0+guest_reserve))) + mem =3D int(math.floor(host_mem*overcommit/vmsc)) + + if os.popen("uname -i").readline().startswith("i386"): + logging.debug("Host is i386 architecture, max guest mem is= 2G") + # Guest system with qemu overhead (64M) can't have more th= an 2G + if mem > 2048 - 64: + vmsc =3D int(math.ceil((host_mem*overcommit)/(2048 - 6= 4.0))) + mem =3D int(math.floor(host_mem*overcommit/vmsc)) + + + logging.info("overcommit =3D %f" % (overcommit)) + logging.info("true overcommit =3D %f " % (float(vmsc*mem) /=20 float(host_mem))) + logging.info("host mem =3D %dM" % (host_mem)) + logging.info("mem =3D %dM" % (mem)) + logging.info("swap =3D %dM" %\ + (int(os.popen("cat /proc/meminfo | grep SwapTotal")\ + .readline().split()[1]) / 1024)) + logging.info("max_vms =3D %d" % (max_vms)) + logging.info("vmsc =3D %d" % (vmsc)) + + # Generate unique keys for random series + skeys =3D [] + dkeys =3D [] + for i in range(0, max(vmsc, max_alloc)): + key =3D "%03s" % (random.randrange(0,999)) + while key in skeys: + key =3D "%03s" % (random.randrange(0,999)) + skeys.append(key) + + key =3D "%03s" % (random.randrange(0,999)) + while key in dkeys: + key =3D "%03s" % (random.randrange(0,999)) + dkeys.append(key) + + lvms =3D [] + lsessions =3D [] + lallocators =3D [] + alloc_port =3D 31284 + + # As we don't know the number and memory amount of VMs in advance,= =20 we need + # to specify and create them here (FIXME: not a nice thing) + params['mem'] =3D mem + params['vms'] =3D params.get("main_vm") + # ksm_size: amount of memory used by allocator + ksm_size =3D mem - guest_reserve + logging.info("ksm_size =3D %dM" % (ksm_size)) + + + params['redirs'] +=3D ' alloc0' + params['guest_port_alloc0'] =3D str(alloc_port) + + if (params['ksm_test_size'] =3D=3D "paralel") : + for j in range(1, max_alloc): + params['redirs'] +=3D ' alloc' + str(j) + params['guest_port_alloc' + str(j)] =3D str(alloc_port + j= ) + + # Creating of the first guest + kvm_preprocessing.preprocess_vm(test, params, env, params['vms']) + lvms.append(kvm_utils.env_get_vm(env, params.get("main_vm"))) + if not lvms[0]: + raise error.TestError("VM object not found in environment") + if not lvms[0].is_alive(): + raise error.TestError("VM seems to be dead; Test requires a=20 living VM") + + logging.info("Booting the first guest %s" % lvms[0].name) + + lsessions.append(kvm_utils.wait_for(lvms[0].ssh_login, 360, 0, 2)) + if not lsessions[0]: + raise error.TestFail("Could not log into first guest") + + + lallocators.append(allocator_com(lvms[0].name,=20 lvms[0].redirs[alloc_port])) + if not lallocators[0]: + raise error.TestFail("Could not create allocator_com class for= =20 vm1") + + + + # Creating of other guest systems + for i in range(1, vmsc): + vm_name =3D "vm" + str(i + 1) + # Last VM is later used to run more allocators simultaneously + """for j in range(1, max_alloc): + params['redirs'] +=3D ' alloc' + str(j) + params['guest_port_alloc' + str(j)] =3D str(alloc_port + j= )""" + + lvms.append(lvms[0].clone(vm_name, params)) + kvm_utils.env_register_vm(env, vm_name, lvms[i]) + params['vms'] +=3D " " + vm_name + + logging.info("Booting guest %s" % lvms[i].name) + if not lvms[i].create(): + raise error.TestFail("Cannot create VM %s" % lvms[i].name) + if not lvms[i].is_alive(): + raise error.TestError("VM %s seems to be dead; Test=20 requires a"\ + "living VM" % lvms[i].name) + + lsessions.append(kvm_utils.wait_for(lvms[i].ssh_login, 360, 0,= 2)) + if not lsessions[i]: + raise error.TestFail("Could not log into guest %s" %=20 lvms[i].name) + + lallocators.append(allocator_com(lvms[i].name,\ + lvms[i].redirs[alloc_port])) + if not lallocators[i]: + raise error.TestFail("Could not create allocator_com class= =20 for %s"\ + % (lvms[i].name)) + + + # Let systems take a rest :-) + time.sleep(vmsc * 2) + logging.info(get_stat(lvms)) + + # Copy the allocator.c into guests + pwd =3D os.path.join(os.environ['AUTODIR'],'tests/kvm') + vksmd_src =3D os.path.join(pwd, "allocator.c") + dst_dir =3D "/tmp" + for vm in lvms: + if not vm.scp_to_remote(vksmd_src, dst_dir): + raise error.TestFail("Remote scp failed %s" % (vm.name)) + logging.info("Phase 0 =3D> passed") + + def phase_1(): + """ Inicialize virtual machine """ + logging.info("Starting phase 1: filling with 0") + logging.info("Preparing the guests and fill in pages by zero") + for session in lsessions: + vm =3D lvms[lsessions.index(session)] + allocator =3D lallocators[lsessions.index(session)] + # Build the test suite + ret =3D session.get_command_status("gcc -o /tmp/allocator = "\ + "/tmp/allocator.c",\ + timeout=3D300) + if ret =3D=3D None or ret: + raise error.TestFail("Failed to build vksmd in the %s"= \ + % (vm.name)) + + # Start the daemon + ret =3D session.get_command_status("/tmp/allocator %d %d" = %=20 (ksm_size,\ +=20 alloc_port)) + if ret =3D=3D None: + raise error.TestFail("Could not run vksmd in guest %s"= \ + % (vm.name)) + if ret: + raise error.TestFail("Could not run vksmd in %s errno:= %d"\ + % (vm.name, ret)) + + ret =3D session.get_command_status("iptables -F;"\ + "iptables -P INPUT ACCEPT= ;") + + allocator.connect() + allocator.recv((ksm_size / 200), 100) + + # Let kksmd works (until shared mem rich expected value) + shm =3D 0 + i =3D 0 + cmd =3D "cat /proc/%d/statm" % vm.pid + while shm < ksm_size: + if i > 64: + logging.info(get_stat(lvms)) + raise error.TestError("SHM didn't merged the memor= y=20 until "\ + "the DL") + logging.debug("Sleep(%d)" % (ksm_size / 200)) + time.sleep(ksm_size / 200) + try: + shm =3D int(os.popen(cmd).readline().split()[2]) + shm =3D shm * 4 / 1024 + i =3D i + 1 + except: + raise error.TestError("Could not fetch shmem info=20 from " + "the /proc") + + # Keep some reserve + time.sleep(ksm_size / 200) + + # Set allocator keys + for i in range(0, vmsc): + lallocators[i].send("init", "%s%s" % (skeys[i], dkeys[i])) + lallocators[i].recv(1, 10) + logging.info(get_stat(lvms)) + logging.info("Phase 1 =3D> passed") + + def phase_2(): + """ Separate first guest memory by generate a special random=20 series """ + logging.info("Starting phase 2: Split the pages on the first=20 guest") + + lallocators[0].send("srandom") + out =3D lallocators[0].recv(ksm_size / 500, 50) + out =3D int(out.split()[4]) + logging.info("PERFORMANCE: %dMB * 1000 / %dms =3D %dMB/s"\ + % (ksm_size, out, (ksm_size * 1000 / out))) + logging.info(get_stat(lvms)) + logging.info("Phase 2 =3D> passed") + + def phase_3(): + """ Sequentional split of pages on guests up to memory limit "= "" + logging.info("Starting phase 3a: Sequentional split of pages o= n=20 guests up "\ + "to memory limit") + last_vm =3D 0 + for i in range(1, vmsc): + vm =3D lvms[i] + session =3D lsessions[i] + allocator =3D lallocators[i] + + allocator.send("srandom") + out =3D "" + while not out.startswith("PASS") and not=20 out.startswith("FAIL"): + free_mem =3D int(os.popen("grep MemFree /proc/meminfo"= )\ + .readline().split()[1]) + logging.debug("FreeMem =3D %d" % (free_mem)) + # We need to keep some memory for python to run. + if free_mem < 32000: + logging.debug("Only %s free memory, killing 0 - %d= =20 hosts"\ + % (free_mem, (i-1))) + for j in range(0, i): + lvms[j].destroy(gracefully =3D False) + last_vm =3D i + break + out =3D allocator._recv() + if last_vm !=3D 0: + break + + allocator.recv(mem / 500, 50) + logging.info("Memory filled by the guest %s" % (vm.name)) + logging.info("Phase 3a =3D> passed") + + """ Check if memory in max loading guest is allright""" + logging.info("Starting phase 3b") + allocator.send("srverify") + allocator.recv(mem / 200, 50) + allocator.disconnect() + # We are going to use the last VM later + if i !=3D (vmsc): + session.close() + vm.destroy(gracefully =3D False) + for i in range(last_vm + 1, vmsc): + lallocators[i].send("verify") + lallocators[i].recv(mem / 200, 50) + lallocators[i].disconnect() + # We are going to use the last VM later + if i !=3D (vmsc - 1): + lsessions[i].close() + lvms[i].destroy(gracefully =3D False) + logging.info(get_stat([lvms[i]])) + logging.info("Phase 3b =3D> passed") + + def phase_4(): + """ Paralel page spliting """ + logging.info("Phase 4: Paralel page spliting") + # We have to wait until allocator is finished (it waits 5=20 seconds to clean + # the socket + + session =3D lsessions[0] + vm =3D lvms[0] + + ret =3D session.get_command_status("gcc -o /tmp/allocator "\ + "/tmp/allocator.c",\ + timeout=3D300) + if ret =3D=3D None or ret: + raise error.TestFail("Failed to build vksmd in the %s"\ + % (vm.name)) + + for all in lallocators: + if all.isConnected(): + all.disconnect() + + del lallocators[:] + ret =3D session.get_command_status("iptables -F;"\ + "iptables -P INPUT ACCEPT;") + + for i in range(0, max_alloc): + ret =3D session.get_command_status("/tmp/allocator %d %d" + % (ksm_size / max_alloc, alloc_port= =20 + i)) + if ret =3D=3D None: + raise error.TestFail("Could not run vksmd in guest %s"= \ + % (vm.name)) + if ret: + raise error.TestFail("Could not run allocator in %s=20 errno: %d"\ + % (vm.name, ret)) + + lallocators.append(allocator_com(vm.name,\ + vm.redirs[alloc_port + i]= )) + if not lallocators[i]: + raise error.TestFail("Could not create allocator_com=20 class for"\ + " %s" % (vm.name)) + + logging.info("Phase 4a: Simultaneous merging") + for i in range(0, max_alloc): + lallocators[i].connect() + + for i in range(0, max_alloc): + lallocators[i].recv((ksm_size / 200), 100) + # Wait until kksmd merges the pages (3 x ksm_size / 3) + shm =3D 0 + i =3D 0 + cmd =3D "cat /proc/%d/statm" % vm.pid + while shm < ksm_size: + if i > 64: + logging.info(get_stat(lvms)) + raise error.TestError("SHM didn't merged the memory=20 until DL") + logging.debug("Sleep(%d)" % (ksm_size / 200)) + time.sleep(ksm_size / 200) + try: + shm =3D int(os.popen(cmd).readline().split()[2]) + shm =3D shm * 4 / 1024 + except: + raise error.TestError("Could not fetch shmem info from= =20 proc") + logging.info(get_stat([vm])) + + + logging.info("Phases 4b: Simultaneous spliting") + # Set keys + for i in range(0, max_alloc): + lallocators[i].send("init", "%s%s" % (skeys[i], dkeys[i])) + lallocators[i].recv(1, 10) + + # Actual splitting + for i in range(0, max_alloc): + lallocators[i].send("srandom") + + for i in range(0, max_alloc): + out =3D lallocators[i].recv(ksm_size / 500, 50) + out =3D int(out.split()[4]) + logging.info("PERFORMANCE: %dMB * 1000 / %dms =3D %dMB/s"\ + % (ksm_size, out, (ksm_size * 1000 / out /=20 max_alloc))) + logging.info(get_stat([vm])) + + logging.info("Phase 4c: Simultaneous verification") + for i in range(0, max_alloc): + lallocators[i].send("srverify") + for i in range(0, max_alloc): + lallocators[i].recv(mem / 200, 50) + logging.info(get_stat([vm])) + + logging.info("Phase 4 =3D> passed") + # Clean-up + for i in range(0, max_alloc): + lallocators[i].disconnect() + session.close() + vm.destroy(gracefully =3D False) + + if params['ksm_test_size'] =3D=3D "paralel": + phase_4() + elif params['ksm_test_size'] =3D=3D "serial": + phase_1() + phase_2() + phase_3() + def run_linux_s3(test, params, env): """ --=20 1.6.2.5