From mboxrd@z Thu Jan 1 00:00:00 1970 From: Amos Kong Subject: [RFC PATCH 4/4] virt: Introduce regression testing infrastructure Date: Fri, 23 Dec 2011 18:28:53 +0800 Message-ID: <20111223102853.29662.53458.stgit@dhcp-8-167.nay.redhat.com> References: <20111223102308.29662.59520.stgit@dhcp-8-167.nay.redhat.com> Mime-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit To: lmr@redhat.com, wquan@redhat.com, kvm@vger.kernel.org, jasowang@redhat.com, rhod@redhat.com, autotest@test.kernel.org Return-path: Received: from mx1.redhat.com ([209.132.183.28]:55137 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1756551Ab1LWK3O (ORCPT ); Fri, 23 Dec 2011 05:29:14 -0500 In-Reply-To: <20111223102308.29662.59520.stgit@dhcp-8-167.nay.redhat.com> Sender: kvm-owner@vger.kernel.org List-ID: regression.py: 'regression' module is used to compare the test results of two jobs, we can use it (regression.compare()) at the end of control file, This script can also be used directly. Example: | # python regression.py ntttcp /ntttcp-result1 /ntttcp-result2 \ | ../../tests/kvm/perf.conf | Fri Dec 23 17:23:08 2011 | | 1 - /tmp/netperf-avg-0.385058442362.txt | 2 - /tmp/netperf-avg-0.66384166902.txt | | ======================== | buf(k)| throughput(Mbit/s) | 1 2| 109.548 | 2 2| 104.239 | % | -4.8 | 1 4| 209.519 | 2 4| 211.633 | % | +1.0 analyzer.py: It's used to compare two test results (standard format), it can also be used directly. | # python analyzer.py /result1.RHS /ntttcp-result2.RHS perf.conf: config test related parameters. It supports to compare current result with the result in autotest server. autotest result directory should be shared by NFS first, and specify its address in perf.conf Signed-off-by: Amos Kong --- client/tests/kvm/control | 7 + client/tests/kvm/perf.conf | 23 ++++ client/virt/tests/analyzer.py | 224 +++++++++++++++++++++++++++++++++++++++ client/virt/tests/regression.py | 33 ++++++ 4 files changed, 287 insertions(+), 0 deletions(-) create mode 100644 client/tests/kvm/perf.conf create mode 100644 client/virt/tests/analyzer.py create mode 100644 client/virt/tests/regression.py diff --git a/client/tests/kvm/control b/client/tests/kvm/control index 950154c..5cdf506 100644 --- a/client/tests/kvm/control +++ b/client/tests/kvm/control @@ -67,3 +67,10 @@ if args: parser.parse_string(str) virt_utils.run_tests(parser, job) + +# compare the perfmance results of job +# from autotest_lib.client.virt.tests import regression +# regression.compare("ntttcp", "$olddir", +# "%s/results/default/" % os.environ['AUTODIR'], +# config_file="%s/tests/kvm/perf.conf" % os.environ['AUTODIR'], +# output_dir="%s/results/default/" % os.environ['AUTODIR']) diff --git a/client/tests/kvm/perf.conf b/client/tests/kvm/perf.conf new file mode 100644 index 0000000..31b72b2 --- /dev/null +++ b/client/tests/kvm/perf.conf @@ -0,0 +1,23 @@ +# this config file is used to set test related parameters +# + +[server] +result_nfs = kvm-autotest.englab.nay.redhat.com:/usr/local/autotest/results +result_mntdir = /results/ + +[ntttcp] +result_dir = results +result_file_pattern = .*.RHS + +[netperf] +result_dir = results +result_file_pattern = netperf-result.RHS + +[iozone] +result_dir = guest_test_results +result_file_pattern = + +[ffsb] +result_dir = results +result_file_pattern = + diff --git a/client/virt/tests/analyzer.py b/client/virt/tests/analyzer.py new file mode 100644 index 0000000..9023c77 --- /dev/null +++ b/client/virt/tests/analyzer.py @@ -0,0 +1,224 @@ +import sys, re, string, time, commands, os, random + +def aton(str): + substring = re.split("\.", str) + if len(substring) == 1: + if substring[0].isdigit(): + return string.atoi(str) + elif len(substring) == 2: + if substring[0].isdigit() and substring[1].isdigit(): + return string.atof(str) + return False + +def avg(dict, i): + linetmp = [] + tmp = [] + lines = {} + + filecounts = len(dict) + for j in range(len(dict)): + lines[j] = re.split("\|", dict[j][i]) + for value in range(len(lines[0])): + avgtmp = 0 + column_caculate = 2 + if value < column_caculate: + linetmp.append(lines[0][value]) + else: + space = "" + strlen = len(lines[0][value]) + for i in range(len(lines)): + avgtmp += (aton(lines[i][value].strip())) + if len(re.findall("\.", lines[0][value])) == 0: + avgtmpstr = "%d" % (avgtmp/filecounts) + else: + avgtmpstr = "%.2f" % (avgtmp/filecounts) + + strlenvalue = len(avgtmpstr) + tmplen = strlen-strlenvalue + if value == (len(lines[0])-1): + for v in range(tmplen-1): + space += " " + avgtmpstr= space + avgtmpstr + "\n" + linetmp.append(avgtmpstr) + break + for v in range(tmplen): + space += " " + avgtmpstr = space + avgtmpstr + linetmp.append(avgtmpstr) + line = "|".join(linetmp) + return line + +def avgfile(filenames): + """ + caculate the average of namelist + 1)get the data of every file, then put the data into the dict + 2)caculat the average of the file + """ + filelines = [] + dict = {} + name = "/tmp/netperf-avg-%s.txt" % random.random() + + for i in range(len(filenames)): + fd = open(filenames[i], "r") + dict[i] = fd.readlines() + fd.close() + filenum = len(dict) + if filenum == 1: + content = dict[0] + else: + for i in range(len(dict[0])): + if dict[0][i] == dict[1][i]: + filelines.append(dict[0][i]) + else: + line = avg(dict, i) + filelines.append(line) + content = filelines + f = open(name, "w") + f.write(''.join(content)) + f.close() + return name + +def record_result(name1, name2, file): + + def tee(content): + f = open(file, "a") + f.write(content + "\n") + print content + + result1 = {} + result2 = {} + result3 = {} + row = 0 + strlen = 0 + eachLine = "" + tee(name1) + + # read the first file + fd = open(name1, "r") + for eachLine in fd: + #eachLine = ''.join(eachLine.split()) + eachLine = eachLine.replace('\r', '') + eachLine = eachLine.replace('\n', '') + result1[row] = re.split("\|", eachLine) + row += 1 + + fd.close() + row = 0 + # read the second file + fd = open(name2, "r") + for eachLine in fd: + #eachLine = ''.join(eachLine.split()) + eachLine = eachLine.replace('\r', '') + eachLine = eachLine.replace('\n', '') + if re.findall("sessions", eachLine) != 0: + strlen = len(eachLine) + result2[row] = re.split("\|", eachLine) + row += 1 + + fd.close() + + name1_list = re.split("/", name1) + name2_list = re.split("/", name2) + + len1 = len(name1_list) + file_name11 = name1_list[len1-1] + len2 = len(name2_list) + file_name22 = name2_list[len2-1] + + #rename the file which will save the result + name1list = re.split("-", file_name11) + name2list = re.split("-", file_name22) + if (len(name1list) > len(name2list)): + namelen = len(name2list) + else: + namelen = len(name1list) + + resultlist = [] + for i in range(namelen): + if name1list[i] == name2list[i]: + resultlist.append(name1list[i]) + + timevalue = time.time() + timestring = time.ctime(timevalue) + tee("%s\n" % timestring) + tee("1 - %s" % name1) + tee("2 - %s\n" % name2) + + #caculate the length of each line + eachLine = "" + for i in range(strlen): + eachLine += "=" + eachLine += "======" + tee("%s" % eachLine) + row = strlen = 0 + for row in result1: + if result1[row] == result2[row]: + if len(result1[row]) > 1: + result1[row][0] = " %s" % result1[row][0] + eachLine = "|".join(result1[row]) + tee("%s" % eachLine) + else: + eachLine = "|".join(result1[row]) + tee("%s" % eachLine) + else: + strlen = len(result1[row][0]) + tmp = result1[row][0].strip() + tmp = "%s" % tmp + result1[row][0] = tmp.rjust(strlen, ' ') + result1[row][0] = "1 %s" % result1[row][0] + eachLine = "|".join(result1[row]) + tee("%s" % eachLine) + + strlen = len(result2[row][0]) + tmp = result2[row][0].strip() + tmp = "%s" % tmp + result2[row][0] = tmp.rjust(strlen, ' ') + result2[row][0] = "2 %s" % result2[row][0] + eachLine = "|".join(result2[row]) + tee("%s" % eachLine) + + result_tmp = [] + strlen = 0 + result_colum = 1 + for i in range(len(result1[row])): + if i < result_colum: + tmp_str = "" + strlen += len(result1[row][i]) + tmp_str = tmp_str.rjust(strlen-1, ' ') + tmp_str = "%" + tmp_str + if i == result_colum - 1: + result_tmp.append(tmp_str) + elif i >= result_colum: + strlen = len(result1[row][i]) + aa = (result1[row][i]).strip() + aa = string.atof(aa) + bb = (result2[row][i]).strip() + bb = string.atof(bb) + if aa != 0: + cc = ((bb-aa)/aa)*100 + if cc > 0: + result = "+%.1f" % cc + else: + result = "%.1f" % cc + else: + result = "0" + result_str = result.rjust(strlen, ' ') + result_tmp.append(result_str) + + eachLine = "|".join(result_tmp) + tee("%s" % eachLine) + +def analyze(list_files1, list_files2, output_dir=""): + average1 = avgfile(list_files1.split()) + average2 = avgfile(list_files2.split()) + f = os.path.join(output_dir, "end-report-%s.txt" % + time.strftime('%Y-%m-%d-%H.%M.%S')) + record_result(average1, average2, f) + commands.getoutput("rm -f /tmp/netperf-avg-*") + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print 'Usage: python %s "$results list1" "results list2"' % sys.argv[0] + sys.exit(1) + analyze(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/client/virt/tests/regression.py b/client/virt/tests/regression.py new file mode 100644 index 0000000..e2588a7 --- /dev/null +++ b/client/virt/tests/regression.py @@ -0,0 +1,33 @@ +import ConfigParser, sys, commands, os +import analyzer + +def compare(testname, olddir, curdir, config_file='perf.conf', output_dir=""): + config = ConfigParser.ConfigParser() + config.read(config_file) + + result_nfs = config.get("server", "result_nfs") + result_mntdir = config.get("server", "result_mntdir") + result_dir = config.get(testname, "result_dir") + result_file_pattern = config.get(testname, "result_file_pattern") + + def search_files(dir): + cmd = 'find %s|grep %s|grep "%s/%s"' % (dir, + testname, result_dir, result_file_pattern) + return commands.getoutput(cmd) + + if not os.path.isdir(result_mntdir): + os.mkdir(result_mntdir) + commands.getoutput("mount %s %s" % (result_nfs, result_mntdir)) + + if not os.path.isabs(olddir): + olddir = result_mntdir + olddir + oldlist = search_files(olddir) + newlist = search_files(curdir) + if oldlist != "" or newlist != "": + analyzer.analyze(oldlist, newlist, output_dir) + +if __name__ == "__main__": + if len(sys.argv) != 5: + print 'Usage: python %s $testname $dir1 $dir2 $configfile' % sys.argv[0] + sys.exit(1) + compare(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])