[PATCH 0/3] Toaster improvements (OE-Core part)

[PATCH 0/3] Toaster improvements (OE-Core part)

[Paul Eggleton]

Note: this should be merged into OE-Core at the same time that the
corresponding changes to BitBake are merged.


The following changes since commit 9f69e00200cdbd5ba2e46a54f33c29797816e43f:

  runqemu: remove core-image-* whitelist (2013-12-09 18:01:39 +0000)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib paule/toaster-oecore
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/toaster-oecore

Alexandru DAMIAN (3):
  toaster.bbclass: read layer information
  toaster.bbclass: read build stats
  toaster.bbclass: read package and image information

 meta/classes/toaster.bbclass | 230 +++++++++++++++++++++++++++++++++++++++----
 1 file changed, 213 insertions(+), 17 deletions(-)

-- 
1.8.1.2

[PATCH 1/3] toaster.bbclass: read layer information

[Paul Eggleton]

From: Alexandru DAMIAN <alexandru.damian@intel.com>

In the process of removing the local system
accesses from toaster UI (which must be able to
run remotely), the code to read layer information
is moved from Bitbake Toaster UI
to the server-side toaster.bbclass

    [YOCTO #5604]

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
 meta/classes/toaster.bbclass | 68 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 67 insertions(+), 1 deletion(-)

diff --git a/meta/classes/toaster.bbclass b/meta/classes/toaster.bbclass
index 8dc1663..ec9b6c5 100644
--- a/meta/classes/toaster.bbclass
+++ b/meta/classes/toaster.bbclass
@@ -24,7 +24,73 @@
 #
 #
 
-# 1. Dump package file info data
+# Find and dump layer info when we got the layers parsed
+
+
+
+python toaster_layerinfo_dumpdata() {
+    import subprocess
+
+    def _get_git_branch(layer_path):
+        branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0]
+        branch = branch.replace('refs/heads/', '').rstrip()
+        return branch
+
+    def _get_git_revision(layer_path):
+        revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip()
+        return revision
+
+    def _get_url_map_name(layer_name):
+        """ Some layers have a different name on openembedded.org site,
+            this method returns the correct name to use in the URL
+        """
+
+        url_name = layer_name
+        url_mapping = {'meta': 'openembedded-core'}
+
+        for key in url_mapping.keys():
+            if key == layer_name:
+                url_name = url_mapping[key]
+
+        return url_name
+
+    def _get_layer_version_information(layer_path):
+
+        layer_version_info = {}
+        layer_version_info['branch'] = _get_git_branch(layer_path)
+        layer_version_info['commit'] = _get_git_revision(layer_path)
+        layer_version_info['priority'] = 0
+
+        return layer_version_info
+
+
+    def _get_layer_dict(layer_path):
+
+        layer_info = {}
+        layer_name = layer_path.split('/')[-1]
+        layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/'
+        layer_url_name = _get_url_map_name(layer_name)
+
+        layer_info['name'] = layer_name
+        layer_info['local_path'] = layer_path
+        layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name)
+        layer_info['version'] = _get_layer_version_information(layer_path)
+
+        return layer_info
+
+
+    bblayers = e.data.getVar("BBLAYERS", True)
+
+    llayerinfo = {}
+
+    for layer in { l for l in bblayers.strip().split(" ") if len(l) }:
+        llayerinfo[layer] = _get_layer_dict(layer)
+
+
+    bb.event.fire(bb.event.MetadataEvent("LayerInfo", llayerinfo), e.data)
+}
+
+# Dump package file info data
 
 python toaster_package_dumpdata() {
     """
-- 
1.8.1.2

[PATCH 2/3] toaster.bbclass: read build stats

[Paul Eggleton]

From: Alexandru DAMIAN <alexandru.damian@intel.com>

In the process of removing the local system
accesses from toaster UI (which must be able to
run remotely), the code to read build stats
is moved from Bitbake Toaster UI
to the server-side toaster.bbclass

The code will accumulate a list of stat files
to be read at build completion. When the
build completes, the whole data list is read and
sent through in a single event.

    [YOCTO #5604]

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
 meta/classes/toaster.bbclass | 73 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)

diff --git a/meta/classes/toaster.bbclass b/meta/classes/toaster.bbclass
index ec9b6c5..59c61c5 100644
--- a/meta/classes/toaster.bbclass
+++ b/meta/classes/toaster.bbclass
@@ -153,6 +153,79 @@ python toaster_image_dumpdata() {
 }
 
 
+
+# collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data
+
+python toaster_collect_task_stats() {
+    import bb.build
+    import bb.event
+    import bb.data
+    import bb.utils
+    import os
+
+    def _append_read_list(v):
+        lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True)
+
+        with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "a") as fout:
+            bn = get_bn(e)
+            bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn)
+            taskdir = os.path.join(bsdir, e.data.expand("${PF}"))
+            fout.write("%s:%s:%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task)))
+
+        bb.utils.unlockfile(lock)
+
+    def _read_stats(filename):
+        cpu_usage = 0
+        disk_io = 0
+        startio = ''
+        endio = ''
+        pn = ''
+        taskname = ''
+        statinfo = {}
+
+        with open(filename, 'r') as task_bs:
+            for line in task_bs.readlines():
+                k,v = line.strip().split(": ", 1)
+                statinfo[k] = v
+
+        try:
+            cpu_usage = statinfo["CPU usage"]
+            endio = statinfo["EndTimeIO"]
+            startio = statinfo["StartTimeIO"]
+        except KeyError:
+            pass    # we may have incomplete data here
+
+        if startio and endio:
+            disk_io = int(endio.strip('\n ')) - int(startio.strip('\n '))
+
+        if cpu_usage:
+            cpu_usage = float(cpu_usage.strip('% \n'))
+
+        return {'cpu_usage': cpu_usage, 'disk_io': disk_io}
+
+
+    if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)):
+        _append_read_list(e)
+        pass
+
+
+    if isinstance(e, bb.event.BuildCompleted):
+        events = []
+        with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "r") as fin:
+            for line in fin:
+                (taskfile, taskname, filename) = line.strip().split(":")
+                events.append((taskfile, taskname, _read_stats(filename)))
+        bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data)
+        os.unlink(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"))
+}
+
+
+# set event handlers
+addhandler toaster_layerinfo_dumpdata
+toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted"
+
+addhandler toaster_collect_task_stats
+toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed"
 do_package[postfuncs] += "toaster_package_dumpdata "
 
 do_rootfs[postfuncs] += "toaster_image_dumpdata "
-- 
1.8.1.2

[PATCH 3/3] toaster.bbclass: read package and image information

[Paul Eggleton]

From: Alexandru DAMIAN <alexandru.damian@intel.com>

In the process of removing the local system accesses
from toaster UI (which must be able to run remotely),
the code to read package information  is moved
from Bitbake Toaster UI to the server-side
toaster.bbclass

     [YOCTO #5604]

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
 meta/classes/toaster.bbclass | 91 +++++++++++++++++++++++++++++++++++---------
 1 file changed, 74 insertions(+), 17 deletions(-)

diff --git a/meta/classes/toaster.bbclass b/meta/classes/toaster.bbclass
index 59c61c5..e1a93b5 100644
--- a/meta/classes/toaster.bbclass
+++ b/meta/classes/toaster.bbclass
@@ -92,6 +92,23 @@ python toaster_layerinfo_dumpdata() {
 
 # Dump package file info data
 
+def _toaster_load_pkgdatafile(dirpath, filepath):
+    import json
+    pkgdata = {}
+    with open(os.path.join(dirpath, filepath), "r") as fin:
+        for line in fin:
+            try:
+                kn, kv = line.strip().split(": ", 1)
+                kn = "_".join([x for x in kn.split("_") if x.isupper()])
+                pkgdata[kn] = kv.strip()
+                if kn == 'FILES_INFO':
+                    pkgdata[kn] = json.loads(kv)
+
+            except ValueError:
+                pass    # ignore lines without valid key: value pairs
+    return pkgdata
+
+
 python toaster_package_dumpdata() {
     """
     Dumps the data created by emit_pkgdata
@@ -103,27 +120,12 @@ python toaster_package_dumpdata() {
 
     pkgdatadir = d.getVar('PKGDESTWORK', True)
 
-
     # scan and send data for each package
-    import json
 
     lpkgdata = {}
     for pkg in packages.split():
 
-        subdata_file = pkgdatadir + "/runtime/%s" % pkg
-        lpkgdata = {}
-
-        sf = open(subdata_file, "r")
-        line = sf.readline()
-        while line:
-            (n, v) = line.rstrip().split(":", 1)
-            if pkg in n:
-                n = n.replace("_" + pkg, "")
-            if n == 'FILES_INFO':
-                lpkgdata[n] = json.loads(v)
-            else:
-                lpkgdata[n] = v.strip()
-            line = sf.readline()
+        lpkgdata = _toaster_load_pkgdatafile(pkgdatadir + "/runtime/", pkg)
 
         # Fire an event containing the pkg data
         bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d)
@@ -209,7 +211,7 @@ python toaster_collect_task_stats() {
         pass
 
 
-    if isinstance(e, bb.event.BuildCompleted):
+    if isinstance(e, bb.event.BuildCompleted) and os.path.exists(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")):
         events = []
         with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "r") as fin:
             for line in fin:
@@ -219,6 +221,58 @@ python toaster_collect_task_stats() {
         os.unlink(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"))
 }
 
+# dump relevant build history data as an event when the build is completed
+
+python toaster_buildhistory_dump() {
+    import re
+    BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory")
+    BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR)
+    pkgdata_dir = e.data.getVar("PKGDATA_DIR", True)
+
+
+    # scan the build targets for this build
+    images = {}
+    allpkgs = {}
+    for target in e._pkgs:
+        installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target))
+        if os.path.exists(installed_img_path):
+            images[target] = {}
+            with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin:
+                for line in fin:
+                    line = line.rstrip(";")
+                    psize, px = line.split("\t")
+                    punit, pname = px.split(" ")
+                    # this size is "installed-size" as it measures how much space it takes on disk
+                    images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []}
+
+            with open("%s/depends.dot" % installed_img_path, "r") as fin:
+                p = re.compile(r' -> ')
+                dot = re.compile(r'.*style=dotted')
+                for line in fin:
+                    line = line.rstrip(';')
+                    linesplit = p.split(line)
+                    if len(linesplit) == 2:
+                        pname = linesplit[0].rstrip('"').strip('"')
+                        dependsname = linesplit[1].split(" ")[0].strip().strip(";").strip('"').rstrip('"')
+                        deptype = "depends"
+                        if dot.match(line):
+                            deptype = "recommends"
+                        if not pname in images[target]:
+                            images[target][pname] = {'size': 0, 'depends' : []}
+                        if not dependsname in images[target]:
+                            images[target][dependsname] = {'size': 0, 'depends' : []}
+                        images[target][pname]['depends'].append((dependsname, deptype))
+
+            for pname in images[target]:
+                if not pname in allpkgs:
+                    allpkgs[pname] = _toaster_load_pkgdatafile("%s/runtime-reverse/" % pkgdata_dir, pname)
+
+
+    data = { 'pkgdata' : allpkgs, 'imgdata' : images }
+
+    bb.event.fire(bb.event.MetadataEvent("ImagePkgList", data), e.data)
+
+}
 
 # set event handlers
 addhandler toaster_layerinfo_dumpdata
@@ -226,6 +280,9 @@ toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted"
 
 addhandler toaster_collect_task_stats
 toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed"
+
+addhandler toaster_buildhistory_dump
+toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted"
 do_package[postfuncs] += "toaster_package_dumpdata "
 
 do_rootfs[postfuncs] += "toaster_image_dumpdata "
-- 
1.8.1.2