From mboxrd@z Thu Jan 1 00:00:00 1970 From: Lucas Meneghel Rodrigues Subject: Re: [Autotest] [KVM_AUTOTEST][PATCH] KSM_overcommit: dynamic reserve calculation Date: Thu, 06 May 2010 15:38:25 -0300 Message-ID: <1273171105.15652.138.camel@freedom> References: <4BE1DAAA.80508@redhat.com> Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: QUOTED-PRINTABLE Cc: KVM list , Autotest mailing list , Jiri Zupka , Jason Wang To: =?UTF-8?Q?Luk=C3=A1=C5=A1?= Doktor Return-path: Received: from mx1.redhat.com ([209.132.183.28]:7820 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1751228Ab0EFSia (ORCPT ); Thu, 6 May 2010 14:38:30 -0400 In-Reply-To: <4BE1DAAA.80508@redhat.com> Sender: kvm-owner@vger.kernel.org List-ID: On Wed, 2010-05-05 at 21:52 +0100, Luk=C3=A1=C5=A1 Doktor wrote: > Hi, >=20 > we are back with new features of KSM_overcommit test: > * NEW: guest_reserve and host_reserve are now calculated based on use= d=20 > memory > * NEW: tmpfs reserve is also evaluated to fit the overhead > * NEW: VM alive check during split_guest() > * FIX: In function split_guest() we used incorrect session > * MOD: Increase number of VNC ports Ok guys, thanks for working on this automatic evaluation of reserved memory for host and guest! >=20 > host_reserve: > * Still possible to set this value in cfg > * Calculation: > 1) host_reserve =3D Available memory + minimal guest(128MB) > 2) host_reserve +=3D number of guests * 64 >=20 > guest_reserve: > * Still possible to set this value in cfg > * Calculation: > 1) guest_reserve =3D 256 > 2) guest_reserve +=3D guest memory * constant > ... where constant represents (TMPFS overhead and multiplicative OS=20 > memory consumption) per MB Ok, this sounds good, I'll give my round of testing on it after you guy= s re-send the patch to me with the small comments below addressed. Also, I noticed you guys are not using the full awesomeness of git. Please read the mini tutorial I made so you can use git send-email. http://autotest.kernel.org/wiki/GitWorkflow I made comments on your patch using ------ so it's easier to identify that is a comment and what is the actual patch. Next time, please send the patch using git send-email. >=20 > Tested by ldoktor and jzupka on various HW (2GB-36GB host memory). >=20 diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py index 6bc7987..1d83120 100755 --- a/client/tests/kvm/kvm_vm.py +++ b/client/tests/kvm/kvm_vm.py @@ -396,7 +396,7 @@ class VM: =20 # Find available VNC port, if needed if params.get("display") =3D=3D "vnc": - self.vnc_port =3D kvm_utils.find_free_port(5900, 6000) + self.vnc_port =3D kvm_utils.find_free_port(5900, 6100) =20 # Find random UUID if specified 'uuid =3D random' in confi= g file if params.get("uuid") =3D=3D "random": diff --git a/client/tests/kvm/scripts/allocator.py b/client/tests/kvm/s= cripts/allocator.py index 1036893..227745a 100755 --- a/client/tests/kvm/scripts/allocator.py +++ b/client/tests/kvm/scripts/allocator.py @@ -8,10 +8,12 @@ Auxiliary script used to allocate memory on guests. """ =20 =20 -import os, array, sys, struct, random, copy, inspect, tempfile, dateti= me +import os, array, sys, struct, random, copy, inspect, tempfile, dateti= me, math =20 PAGE_SIZE =3D 4096 # machine page size =20 +TMPFS_OVERHEAD =3D 0.0022 # overhead on 1MB of write data=20 + -----------------------------------------------------------------------= -------- Cool, how did you guys get to this constant? -----------------------------------------------------------------------= -------- class MemFill(object): """ @@ -32,7 +34,8 @@ class MemFill(object): =20 self.tmpdp =3D tempfile.mkdtemp() ret_code =3D os.system("mount -o size=3D%dM tmpfs %s -t tmpfs"= % - ((mem + 25), self.tmpdp)) + ((mem+math.ceil(mem*TMPFS_OVERHEAD)),=20 + self.tmpdp)) if ret_code !=3D 0: if os.getuid() !=3D 0: print ("FAIL: Unable to mount tmpfs " @@ -42,7 +45,7 @@ class MemFill(object): else: self.f =3D tempfile.TemporaryFile(prefix=3D'mem', dir=3Dse= lf.tmpdp) self.allocate_by =3D 'L' - self.npages =3D (mem * 1024 * 1024) / PAGE_SIZE + self.npages =3D ((mem * 1024 * 1024) / PAGE_SIZE) self.random_key =3D random_key self.static_value =3D static_value print "PASS: Initialization" @@ -83,7 +86,7 @@ class MemFill(object): @return: return array of bytes size PAGE_SIZE. """ a =3D array.array("B") - for i in range(PAGE_SIZE / a.itemsize): + for i in range((PAGE_SIZE / a.itemsize)): try: a.append(value) except: diff --git a/client/tests/kvm/tests/ksm_overcommit.py b/client/tests/kv= m/tests/ksm_overcommit.py index 2dd46c4..31d5c61 100644 --- a/client/tests/kvm/tests/ksm_overcommit.py +++ b/client/tests/kvm/tests/ksm_overcommit.py @@ -142,6 +142,10 @@ def run_ksm_overcommit(test, params, env): session =3D None vm =3D None for i in range(1, vmsc): + # Check VMs + for j in range(0, vmsc): + if not lvms[i].is_alive: + raise error.TestFail("one of other VMs is death") -----------------------------------------------------------------------= -------- The message above should be along the lines=20 e_msg =3D "VM %s died while executing static_random_fill on allocator l= oop" % i -----------------------------------------------------------------------= -------- vm =3D lvms[i] session =3D lsessions[i] a_cmd =3D "mem.static_random_fill()" @@ -154,6 +158,8 @@ def run_ksm_overcommit(test, params, env): logging.debug("Watching host memory while filling vm %= s memory", vm.name) while not out.startswith("PASS") and not out.startswit= h("FAIL"): + if not vm.is_alive(): + raise error.TestFail("VM is death") -----------------------------------------------------------------------= -------- The message above also should be along the lines=20 e_msg =3D "VM %s died while executing static_random_fill on allocator l= oop" % i -----------------------------------------------------------------------= -------- free_mem =3D int(utils.read_from_meminfo("MemFree"= )) if (ksm_swap): free_mem =3D (free_mem + @@ -202,7 +208,7 @@ def run_ksm_overcommit(test, params, env): =20 # Verify last machine with randomly generated memory a_cmd =3D "mem.static_random_verify()" - _execute_allocator(a_cmd, lvms[last_vm], session, + _execute_allocator(a_cmd, lvms[last_vm], lsessions[last_vm], (mem / 200 * 50 * perf_ratio)) logging.debug(kvm_test_utils.get_memory_info([lvms[last_vm]])) =20 @@ -338,12 +344,29 @@ def run_ksm_overcommit(test, params, env): =20 # Main test code logging.info("Starting phase 0: Initialization") + # host_reserve: mem reserve kept for the host system to run - host_reserve =3D int(params.get("ksm_host_reserve", 512)) + host_reserve =3D int(params.get("ksm_host_reserve", -1)) + if (host_reserve =3D=3D -1): + # default host_reserve =3D MemAvailable + one_minimal_guest(12= 8MB) + # later we add 64MB per additional guest + host_reserve =3D ((utils.memtotal() - utils.read_from_meminfo(= "MemFree")) + / 1024 + 128) + # using default reserve + _host_reserve =3D 1 -----------------------------------------------------------------------= -------- Be consistent above and assign True instead of 1 -----------------------------------------------------------------------= -------- + else: + _host_reserve =3D False + # guest_reserve: mem reserve kept to avoid guest OS to kill proces= ses - guest_reserve =3D int(params.get("ksm_guest_reserve", 1024)) - logging.debug("Memory reserved for host to run: %d", host_reserve) - logging.debug("Memory reserved for guest to run: %d", guest_reserv= e) + guest_reserve =3D int(params.get("ksm_guest_reserve", -1)) + if (guest_reserve =3D=3D -1): + # default guest_reserve =3D minimal_system_mem(256MB) + # later we add tmpfs overhead + guest_reserve =3D 256 + # using default reserve + _guest_reserve =3D True + else: + _guest_reserve =3D False =20 max_vms =3D int(params.get("max_vms", 2)) overcommit =3D float(params.get("ksm_overcommit_ratio", 2.0)) @@ -355,6 +378,10 @@ def run_ksm_overcommit(test, params, env): =20 if (params['ksm_mode'] =3D=3D "serial"): max_alloc =3D vmsc + if _host_reserve: + # First round of additional guest reserves + host_reserve +=3D vmsc * 64 + _host_reserve =3D vmsc =20 host_mem =3D (int(utils.memtotal()) / 1024 - host_reserve) =20 @@ -402,6 +429,10 @@ def run_ksm_overcommit(test, params, env): if mem - guest_reserve - 1 > 3100: vmsc =3D int(math.ceil((host_mem * overcommit) / (3100 + guest_reserve))) + if _host_reserve: + host_reserve +=3D (vmsc - _host_reserve) * 64 + host_mem -=3D (vmsc - _host_reserve) * 64 + _host_reserve =3D vmsc mem =3D int(math.floor(host_mem * overcommit / vmsc)) =20 if os.popen("uname -i").readline().startswith("i386"): @@ -410,8 +441,19 @@ def run_ksm_overcommit(test, params, env): if mem > 3100 - 64: vmsc =3D int(math.ceil((host_mem * overcommit) / (3100 - 64.0))) + if _host_reserve: + host_reserve +=3D (vmsc - _host_reserve) * 64 + host_mem -=3D (vmsc - _host_reserve) * 64 + _host_reserve =3D vmsc mem =3D int(math.floor(host_mem * overcommit / vmsc)) =20 + # 0.055 represents OS + TMPFS additional reserve per guest ram MB + if _guest_reserve: + guest_reserve +=3D math.ceil(mem * 0.055) + + logging.debug("Memory reserved for host to run: %d", host_reserve) + logging.debug("Memory reserved for guest to run: %d", guest_reserv= e) + logging.debug("Checking KSM status...") ksm_flag =3D 0 for line in os.popen('ksmctl info').readlines(): diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/= tests_base.cfg.sample index ee83ac2..d3a5982 100644 --- a/client/tests/kvm/tests_base.cfg.sample +++ b/client/tests/kvm/tests_base.cfg.sample @@ -302,9 +302,9 @@ variants: ksm_overcommit_ratio =3D 3 # Max paralel runs machine ksm_parallel_ratio =3D 4 - # Host memory reserve - ksm_host_reserve =3D 512 - ksm_guest_reserve =3D 1024 + # Host memory reserve (default - best fit for used mem) + # ksm_host_reserve =3D 512 + # ksm_guest_reserve =3D 1024 variants: - ksm_serial: ksm_mode =3D "serial"