* [PATCH 1/3] toaster.bbclass: read layer information
2013-12-09 19:06 [PATCH 0/3] Toaster improvements (OE-Core part) Paul Eggleton
@ 2013-12-09 19:06 ` Paul Eggleton
2013-12-09 19:06 ` [PATCH 2/3] toaster.bbclass: read build stats Paul Eggleton
2013-12-09 19:06 ` [PATCH 3/3] toaster.bbclass: read package and image information Paul Eggleton
2 siblings, 0 replies; 4+ messages in thread
From: Paul Eggleton @ 2013-12-09 19:06 UTC (permalink / raw)
To: openembedded-core
From: Alexandru DAMIAN <alexandru.damian@intel.com>
In the process of removing the local system
accesses from toaster UI (which must be able to
run remotely), the code to read layer information
is moved from Bitbake Toaster UI
to the server-side toaster.bbclass
[YOCTO #5604]
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
meta/classes/toaster.bbclass | 68 +++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 67 insertions(+), 1 deletion(-)
diff --git a/meta/classes/toaster.bbclass b/meta/classes/toaster.bbclass
index 8dc1663..ec9b6c5 100644
--- a/meta/classes/toaster.bbclass
+++ b/meta/classes/toaster.bbclass
@@ -24,7 +24,73 @@
#
#
-# 1. Dump package file info data
+# Find and dump layer info when we got the layers parsed
+
+
+
+python toaster_layerinfo_dumpdata() {
+ import subprocess
+
+ def _get_git_branch(layer_path):
+ branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0]
+ branch = branch.replace('refs/heads/', '').rstrip()
+ return branch
+
+ def _get_git_revision(layer_path):
+ revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip()
+ return revision
+
+ def _get_url_map_name(layer_name):
+ """ Some layers have a different name on openembedded.org site,
+ this method returns the correct name to use in the URL
+ """
+
+ url_name = layer_name
+ url_mapping = {'meta': 'openembedded-core'}
+
+ for key in url_mapping.keys():
+ if key == layer_name:
+ url_name = url_mapping[key]
+
+ return url_name
+
+ def _get_layer_version_information(layer_path):
+
+ layer_version_info = {}
+ layer_version_info['branch'] = _get_git_branch(layer_path)
+ layer_version_info['commit'] = _get_git_revision(layer_path)
+ layer_version_info['priority'] = 0
+
+ return layer_version_info
+
+
+ def _get_layer_dict(layer_path):
+
+ layer_info = {}
+ layer_name = layer_path.split('/')[-1]
+ layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/'
+ layer_url_name = _get_url_map_name(layer_name)
+
+ layer_info['name'] = layer_name
+ layer_info['local_path'] = layer_path
+ layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name)
+ layer_info['version'] = _get_layer_version_information(layer_path)
+
+ return layer_info
+
+
+ bblayers = e.data.getVar("BBLAYERS", True)
+
+ llayerinfo = {}
+
+ for layer in { l for l in bblayers.strip().split(" ") if len(l) }:
+ llayerinfo[layer] = _get_layer_dict(layer)
+
+
+ bb.event.fire(bb.event.MetadataEvent("LayerInfo", llayerinfo), e.data)
+}
+
+# Dump package file info data
python toaster_package_dumpdata() {
"""
--
1.8.1.2
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH 2/3] toaster.bbclass: read build stats
2013-12-09 19:06 [PATCH 0/3] Toaster improvements (OE-Core part) Paul Eggleton
2013-12-09 19:06 ` [PATCH 1/3] toaster.bbclass: read layer information Paul Eggleton
@ 2013-12-09 19:06 ` Paul Eggleton
2013-12-09 19:06 ` [PATCH 3/3] toaster.bbclass: read package and image information Paul Eggleton
2 siblings, 0 replies; 4+ messages in thread
From: Paul Eggleton @ 2013-12-09 19:06 UTC (permalink / raw)
To: openembedded-core
From: Alexandru DAMIAN <alexandru.damian@intel.com>
In the process of removing the local system
accesses from toaster UI (which must be able to
run remotely), the code to read build stats
is moved from Bitbake Toaster UI
to the server-side toaster.bbclass
The code will accumulate a list of stat files
to be read at build completion. When the
build completes, the whole data list is read and
sent through in a single event.
[YOCTO #5604]
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
meta/classes/toaster.bbclass | 73 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 73 insertions(+)
diff --git a/meta/classes/toaster.bbclass b/meta/classes/toaster.bbclass
index ec9b6c5..59c61c5 100644
--- a/meta/classes/toaster.bbclass
+++ b/meta/classes/toaster.bbclass
@@ -153,6 +153,79 @@ python toaster_image_dumpdata() {
}
+
+# collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data
+
+python toaster_collect_task_stats() {
+ import bb.build
+ import bb.event
+ import bb.data
+ import bb.utils
+ import os
+
+ def _append_read_list(v):
+ lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True)
+
+ with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "a") as fout:
+ bn = get_bn(e)
+ bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn)
+ taskdir = os.path.join(bsdir, e.data.expand("${PF}"))
+ fout.write("%s:%s:%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task)))
+
+ bb.utils.unlockfile(lock)
+
+ def _read_stats(filename):
+ cpu_usage = 0
+ disk_io = 0
+ startio = ''
+ endio = ''
+ pn = ''
+ taskname = ''
+ statinfo = {}
+
+ with open(filename, 'r') as task_bs:
+ for line in task_bs.readlines():
+ k,v = line.strip().split(": ", 1)
+ statinfo[k] = v
+
+ try:
+ cpu_usage = statinfo["CPU usage"]
+ endio = statinfo["EndTimeIO"]
+ startio = statinfo["StartTimeIO"]
+ except KeyError:
+ pass # we may have incomplete data here
+
+ if startio and endio:
+ disk_io = int(endio.strip('\n ')) - int(startio.strip('\n '))
+
+ if cpu_usage:
+ cpu_usage = float(cpu_usage.strip('% \n'))
+
+ return {'cpu_usage': cpu_usage, 'disk_io': disk_io}
+
+
+ if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)):
+ _append_read_list(e)
+ pass
+
+
+ if isinstance(e, bb.event.BuildCompleted):
+ events = []
+ with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "r") as fin:
+ for line in fin:
+ (taskfile, taskname, filename) = line.strip().split(":")
+ events.append((taskfile, taskname, _read_stats(filename)))
+ bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data)
+ os.unlink(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"))
+}
+
+
+# set event handlers
+addhandler toaster_layerinfo_dumpdata
+toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted"
+
+addhandler toaster_collect_task_stats
+toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed"
do_package[postfuncs] += "toaster_package_dumpdata "
do_rootfs[postfuncs] += "toaster_image_dumpdata "
--
1.8.1.2
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH 3/3] toaster.bbclass: read package and image information
2013-12-09 19:06 [PATCH 0/3] Toaster improvements (OE-Core part) Paul Eggleton
2013-12-09 19:06 ` [PATCH 1/3] toaster.bbclass: read layer information Paul Eggleton
2013-12-09 19:06 ` [PATCH 2/3] toaster.bbclass: read build stats Paul Eggleton
@ 2013-12-09 19:06 ` Paul Eggleton
2 siblings, 0 replies; 4+ messages in thread
From: Paul Eggleton @ 2013-12-09 19:06 UTC (permalink / raw)
To: openembedded-core
From: Alexandru DAMIAN <alexandru.damian@intel.com>
In the process of removing the local system accesses
from toaster UI (which must be able to run remotely),
the code to read package information is moved
from Bitbake Toaster UI to the server-side
toaster.bbclass
[YOCTO #5604]
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
meta/classes/toaster.bbclass | 91 +++++++++++++++++++++++++++++++++++---------
1 file changed, 74 insertions(+), 17 deletions(-)
diff --git a/meta/classes/toaster.bbclass b/meta/classes/toaster.bbclass
index 59c61c5..e1a93b5 100644
--- a/meta/classes/toaster.bbclass
+++ b/meta/classes/toaster.bbclass
@@ -92,6 +92,23 @@ python toaster_layerinfo_dumpdata() {
# Dump package file info data
+def _toaster_load_pkgdatafile(dirpath, filepath):
+ import json
+ pkgdata = {}
+ with open(os.path.join(dirpath, filepath), "r") as fin:
+ for line in fin:
+ try:
+ kn, kv = line.strip().split(": ", 1)
+ kn = "_".join([x for x in kn.split("_") if x.isupper()])
+ pkgdata[kn] = kv.strip()
+ if kn == 'FILES_INFO':
+ pkgdata[kn] = json.loads(kv)
+
+ except ValueError:
+ pass # ignore lines without valid key: value pairs
+ return pkgdata
+
+
python toaster_package_dumpdata() {
"""
Dumps the data created by emit_pkgdata
@@ -103,27 +120,12 @@ python toaster_package_dumpdata() {
pkgdatadir = d.getVar('PKGDESTWORK', True)
-
# scan and send data for each package
- import json
lpkgdata = {}
for pkg in packages.split():
- subdata_file = pkgdatadir + "/runtime/%s" % pkg
- lpkgdata = {}
-
- sf = open(subdata_file, "r")
- line = sf.readline()
- while line:
- (n, v) = line.rstrip().split(":", 1)
- if pkg in n:
- n = n.replace("_" + pkg, "")
- if n == 'FILES_INFO':
- lpkgdata[n] = json.loads(v)
- else:
- lpkgdata[n] = v.strip()
- line = sf.readline()
+ lpkgdata = _toaster_load_pkgdatafile(pkgdatadir + "/runtime/", pkg)
# Fire an event containing the pkg data
bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d)
@@ -209,7 +211,7 @@ python toaster_collect_task_stats() {
pass
- if isinstance(e, bb.event.BuildCompleted):
+ if isinstance(e, bb.event.BuildCompleted) and os.path.exists(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")):
events = []
with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "r") as fin:
for line in fin:
@@ -219,6 +221,58 @@ python toaster_collect_task_stats() {
os.unlink(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"))
}
+# dump relevant build history data as an event when the build is completed
+
+python toaster_buildhistory_dump() {
+ import re
+ BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory")
+ BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR)
+ pkgdata_dir = e.data.getVar("PKGDATA_DIR", True)
+
+
+ # scan the build targets for this build
+ images = {}
+ allpkgs = {}
+ for target in e._pkgs:
+ installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target))
+ if os.path.exists(installed_img_path):
+ images[target] = {}
+ with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin:
+ for line in fin:
+ line = line.rstrip(";")
+ psize, px = line.split("\t")
+ punit, pname = px.split(" ")
+ # this size is "installed-size" as it measures how much space it takes on disk
+ images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []}
+
+ with open("%s/depends.dot" % installed_img_path, "r") as fin:
+ p = re.compile(r' -> ')
+ dot = re.compile(r'.*style=dotted')
+ for line in fin:
+ line = line.rstrip(';')
+ linesplit = p.split(line)
+ if len(linesplit) == 2:
+ pname = linesplit[0].rstrip('"').strip('"')
+ dependsname = linesplit[1].split(" ")[0].strip().strip(";").strip('"').rstrip('"')
+ deptype = "depends"
+ if dot.match(line):
+ deptype = "recommends"
+ if not pname in images[target]:
+ images[target][pname] = {'size': 0, 'depends' : []}
+ if not dependsname in images[target]:
+ images[target][dependsname] = {'size': 0, 'depends' : []}
+ images[target][pname]['depends'].append((dependsname, deptype))
+
+ for pname in images[target]:
+ if not pname in allpkgs:
+ allpkgs[pname] = _toaster_load_pkgdatafile("%s/runtime-reverse/" % pkgdata_dir, pname)
+
+
+ data = { 'pkgdata' : allpkgs, 'imgdata' : images }
+
+ bb.event.fire(bb.event.MetadataEvent("ImagePkgList", data), e.data)
+
+}
# set event handlers
addhandler toaster_layerinfo_dumpdata
@@ -226,6 +280,9 @@ toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted"
addhandler toaster_collect_task_stats
toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed"
+
+addhandler toaster_buildhistory_dump
+toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted"
do_package[postfuncs] += "toaster_package_dumpdata "
do_rootfs[postfuncs] += "toaster_image_dumpdata "
--
1.8.1.2
^ permalink raw reply related [flat|nested] 4+ messages in thread