From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by yocto-www.yoctoproject.org (Postfix, from userid 118) id 3C703E0083D; Mon, 7 Mar 2016 10:39:37 -0800 (PST) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on yocto-www.yoctoproject.org X-Spam-Level: X-Spam-Status: No, score=-6.9 required=5.0 tests=BAYES_00,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 X-Spam-HAM-Report: * -5.0 RCVD_IN_DNSWL_HI RBL: Sender listed at http://www.dnswl.org/, high * trust * [192.55.52.93 listed in list.dnswl.org] * -1.9 BAYES_00 BODY: Bayes spam probability is 0 to 1% * [score: 0.0000] Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by yocto-www.yoctoproject.org (Postfix) with ESMTP id 0569DE0077F for ; Mon, 7 Mar 2016 10:39:33 -0800 (PST) Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP; 07 Mar 2016 10:39:34 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.22,552,1449561600"; d="scan'208";a="928712151" Received: from gmorga1x-mobl.ger.corp.intel.com (HELO [10.252.19.21]) ([10.252.19.21]) by orsmga002.jf.intel.com with ESMTP; 07 Mar 2016 10:39:32 -0800 To: toaster@yoctoproject.org References: <1457353301-698-1-git-send-email-elliot.smith@intel.com> <1457353301-698-2-git-send-email-elliot.smith@intel.com> From: Michael Wood Message-ID: <56DDCAE3.80400@intel.com> Date: Mon, 7 Mar 2016 18:39:31 +0000 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Thunderbird/38.5.1 MIME-Version: 1.0 In-Reply-To: <1457353301-698-2-git-send-email-elliot.smith@intel.com> Subject: Re: [PATCH 1/3] toaster.bbclass: improve how we gather buildstats for Toaster X-BeenThere: toaster@yoctoproject.org X-Mailman-Version: 2.1.13 Precedence: list List-Id: Web based interface for BitBake List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 07 Mar 2016 18:39:37 -0000 Content-Type: text/plain; charset="windows-1252"; format="flowed" Content-Transfer-Encoding: quoted-printable On 07/03/16 12:21, Elliot Smith wrote: > Clean up the code which gathers buildstats for Toaster, > and modify the field names so that the correct parts of the > buildstats files are used to derive the CPU usage values. > > Also derive elapsed time for the build here, rather than in > Toaster, as we have ready access to the data in the correct > format. > > [YOCTO #8842] > > Signed-off-by: Elliot Smith > --- > meta/classes/toaster.bbclass | 70 ++++++++++++++++++++++++++++---------= ------- > 1 file changed, 45 insertions(+), 25 deletions(-) > > diff --git a/meta/classes/toaster.bbclass b/meta/classes/toaster.bbclass > index 51a4c74..b7e8773 100644 > --- a/meta/classes/toaster.bbclass > +++ b/meta/classes/toaster.bbclass > @@ -202,25 +202,37 @@ python toaster_collect_task_stats() { > import bb.utils > import os > = > + toaster_statlist_file =3D os.path.join(e.data.getVar('BUILDSTATS_BAS= E', True), "toasterstatlist") > + > if not e.data.getVar('BUILDSTATS_BASE', True): > return # if we don't have buildstats, we cannot collect stats > = > + def stat_to_float(value): > + return float(value.strip('% \n\r')) > + > def _append_read_list(v): > lock =3D bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.loc= k"), False, True) > = > - with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "= toasterstatlist"), "a") as fout: > + with open(toaster_statlist_file, "a") as fout: > taskdir =3D e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/= ${PF}") > fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname, os= .path.join(taskdir, e.task), e.data.expand("${PN}"))) > = > bb.utils.unlockfile(lock) > = > def _read_stats(filename): > - cpu_usage =3D 0 > - disk_io =3D 0 > - started =3D '0' > - ended =3D '0' > - pn =3D '' > + # seconds > + cpu_time_user =3D 0 > + cpu_time_system =3D 0 > + > + # bytes > + disk_io_read =3D 0 > + disk_io_write =3D 0 > + > + started =3D 0 > + ended =3D 0 > + > taskname =3D '' > + > statinfo =3D {} > = > with open(filename, 'r') as task_bs: > @@ -228,41 +240,49 @@ python toaster_collect_task_stats() { > k,v =3D line.strip().split(": ", 1) > statinfo[k] =3D v > = > - if "CPU usage" in statinfo: > - cpu_usage =3D str(statinfo["CPU usage"]).strip('% \n\r') > - > - if "IO write_bytes" in statinfo: > - disk_io =3D disk_io + int(statinfo["IO write_bytes"].strip('= % \n\r')) > - > - if "IO read_bytes" in statinfo: > - disk_io =3D disk_io + int(statinfo["IO read_bytes"].strip('%= \n\r')) > - > if "Started" in statinfo: > - started =3D str(statinfo["Started"]).strip('% \n\r') > + started =3D stat_to_float(statinfo["Started"]) > = > if "Ended" in statinfo: > - ended =3D str(statinfo["Ended"]).strip('% \n\r') > + ended =3D stat_to_float(statinfo["Ended"]) > = > - elapsed_time =3D float(ended) - float(started) > + if "Child rusage ru_utime" in statinfo: > + cpu_time_user =3D cpu_time_user + stat_to_float(statinfo["Ch= ild rusage ru_utime"]) > = > - cpu_usage =3D float(cpu_usage) > + if "Child rusage ru_stime" in statinfo: > + cpu_time_system =3D cpu_time_system + stat_to_float(statinfo= ["Child rusage ru_stime"]) > = > - return {'cpu_usage': cpu_usage, 'disk_io': disk_io, 'elapsed_tim= e': elapsed_time} > + if "IO write_bytes" in statinfo: > + write_bytes =3D int(statinfo["IO write_bytes"].strip('% \n\r= ')) > + disk_io_write =3D disk_io_write + write_bytes > = > + if "IO read_bytes" in statinfo: > + read_bytes =3D int(statinfo["IO read_bytes"].strip('% \n\r')) > + disk_io_read =3D disk_io_read + read_bytes > + > + return { > + 'stat_file': filename, > + 'cpu_time_user': cpu_time_user, > + 'cpu_time_system': cpu_time_system, > + 'disk_io_read': disk_io_read, > + 'disk_io_write': disk_io_write, > + 'started': started, > + 'ended': ended > + } > = > if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)): > _append_read_list(e) > pass > = > - > - if isinstance(e, bb.event.BuildCompleted) and os.path.exists(os.path= .join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")): > + if isinstance(e, bb.event.BuildCompleted) and os.path.exists(toaster= _statlist_file): > events =3D [] > - with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "= toasterstatlist"), "r") as fin: > + with open(toaster_statlist_file, "r") as fin: > for line in fin: > (taskfile, taskname, filename, recipename) =3D line.str= ip().split("::") > - events.append((taskfile, taskname, _read_stats(filename)= , recipename)) > + stats =3D _read_stats(filename) > + events.append((taskfile, taskname, stats, recipename)) > bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events),= e.data) > - os.unlink(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "= toasterstatlist")) > + #os.unlink(toaster_statlist_file) Did you mean to leave that 'unlink' commented out? > } > = > # dump relevant build history data as an event when the build is comple= ted --------------------------------------------------------------------- Intel Corporation (UK) Limited Registered No. 1134945 (England) Registered Office: Pipers Way, Swindon SN3 1RJ VAT No: 860 2173 47 This e-mail and any attachments may contain confidential material for the sole use of the intended recipient(s). Any review or distribution by others is strictly prohibited. If you are not the intended recipient, please contact the sender and delete all copies.