[PATCH 0/1] Remove unused target tree data for Hob

[PATCH 0/1] Remove unused target tree data for Hob

[Joshua Lock]

Here's a copy of Liping's "Remove unused target tree data for Hob" with my
changes made per my review and my SoB added.

The following changes since commit 09a9146262d58dfe4a2ea4270026b90ae33f6c91:

  parse/ConfHandler: Fix multiline variable corruption (2011-07-21 11:08:19 +0100)

are available in the git repository at:
  git://github.com/incandescant/bitbake lke-speed
  https://github.com/incandescant/bitbake/tree/lke-speed

Liping Ke (1):
  Remove unused target tree data for Hob

 lib/bb/cooker.py |   89 +++++++++++++++++++++++++++++++++++++++++-------------
 1 files changed, 68 insertions(+), 21 deletions(-)

-- 
1.7.6

[PATCH 1/1] Remove unused target tree data for Hob

[Joshua Lock]

From: Liping Ke <liping.ke@intel.com>

Since Hob only needes package dependency information, we can
create a new version of package information retrieving methods,
remove task dependency information, so that we can greatly
reduce data loading time for Hob

Signed-off-by: Liping Ke <liping.ke@intel.com>
Signed-off-by: Joshua Lock <josh@linux.intel.com>
---
 lib/bb/cooker.py |   89 +++++++++++++++++++++++++++++++++++++++++-------------
 1 files changed, 68 insertions(+), 21 deletions(-)

diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 23f2c76..0326b8c 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -299,7 +299,6 @@ class BBCooker:
         """
         # Need files parsed
         self.updateCache()
-
         # If we are told to do the None task then query the default task
         if (task == None):
             task = self.configuration.cmd
@@ -319,18 +318,16 @@ class BBCooker:
             runlist.append([k, "do_%s" % task])
         taskdata.add_unresolved(localdata, self.status)
 
-        rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
-        rq.rqdata.prepare()
-
-        return taskdata, rq
+        return runlist, taskdata
 
-    def generateDepTreeData(self, pkgs_to_build, task, more_meta=False):
+    def generateTaskDepTreeData(self, pkgs_to_build, task):
         """
-        Create a dependency tree of pkgs_to_build, returning the data.
-        When more_meta is set to True include summary, license and group
-        information in the returned tree.
+        Create a dependency graph of pkgs_to_build including reverse dependency
+        information.
         """
-        taskdata, rq = self.prepareTreeData(pkgs_to_build, task)
+        runlist, taskdata = self.prepareTreeData(pkgs_to_build, task)
+        rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
+        rq.rqdata.prepare()
 
         seen_fnids = []
         depend_tree = {}
@@ -348,18 +345,10 @@ class BBCooker:
             fn = taskdata.fn_index[fnid]
             pn = self.status.pkg_fn[fn]
             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
-            if more_meta:
-                summary = self.status.summary[fn]
-                lic = self.status.license[fn]
-                section = self.status.section[fn]
             if pn not in depend_tree["pn"]:
                 depend_tree["pn"][pn] = {}
                 depend_tree["pn"][pn]["filename"] = fn
                 depend_tree["pn"][pn]["version"] = version
-                if more_meta:
-                    depend_tree["pn"][pn]["summary"] = summary
-                    depend_tree["pn"][pn]["license"] = lic
-                    depend_tree["pn"][pn]["section"] = section
             for dep in rq.rqdata.runq_depends[task]:
                 depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
                 deppn = self.status.pkg_fn[depfn]
@@ -403,13 +392,71 @@ class BBCooker:
 
         return depend_tree
 
+    def generatePkgDepTreeData(self, pkgs_to_build, task):
+        """
+        Create a dependency tree of pkgs_to_build, returning the data.
+        """
+        _, taskdata = self.prepareTreeData(pkgs_to_build, task)
+        tasks_fnid = []
+        if len(taskdata.tasks_name) != 0:
+            for task in xrange(len(taskdata.tasks_name)):
+                tasks_fnid.append(taskdata.tasks_fnid[task])
+
+        seen_fnids = []
+        depend_tree = {}
+        depend_tree["depends"] = {}
+        depend_tree["pn"] = {}
+        depend_tree["rdepends-pn"] = {}
+        depend_tree["packages"] = {}
+        depend_tree["rdepends-pkg"] = {}
+
+        for task in xrange(len(tasks_fnid)):
+            fnid = tasks_fnid[task]
+            fn = taskdata.fn_index[fnid]
+            pn = self.status.pkg_fn[fn]
+            version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
+            summary = self.status.summary[fn]
+            lic = self.status.license[fn]
+            section = self.status.section[fn]
+            if pn not in depend_tree["pn"]:
+                depend_tree["pn"][pn] = {}
+                depend_tree["pn"][pn]["filename"] = fn
+                depend_tree["pn"][pn]["version"] = version
+                depend_tree["pn"][pn]["summary"] = summary
+                depend_tree["pn"][pn]["license"] = lic
+                depend_tree["pn"][pn]["section"] = section
+
+            if fnid not in seen_fnids:
+                seen_fnids.append(fnid)
+                packages = []
+
+                depend_tree["depends"][pn] = []
+                for dep in taskdata.depids[fnid]:
+                    depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
+
+                depend_tree["rdepends-pn"][pn] = []
+                for rdep in taskdata.rdepids[fnid]:
+                    depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
+
+                rdepends = self.status.rundeps[fn]
+                for package in rdepends:
+                    packages.append(package)
+
+                for package in packages:
+                    if package not in depend_tree["packages"]:
+                        depend_tree["packages"][package] = {}
+                        depend_tree["packages"][package]["pn"] = pn
+                        depend_tree["packages"][package]["filename"] = fn
+                        depend_tree["packages"][package]["version"] = version
+
+        return depend_tree
 
     def generateDepTreeEvent(self, pkgs_to_build, task):
         """
         Create a task dependency graph of pkgs_to_build.
         Generate an event with the result
         """
-        depgraph = self.generateDepTreeData(pkgs_to_build, task)
+        depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
         bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.configuration.data)
 
     def generateDotGraphFiles(self, pkgs_to_build, task):
@@ -418,7 +465,7 @@ class BBCooker:
         Save the result to a set of .dot files.
         """
 
-        depgraph = self.generateDepTreeData(pkgs_to_build, task)
+        depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
 
         # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
         depends_file = file('pn-depends.dot', 'w' )
@@ -625,7 +672,7 @@ class BBCooker:
             pkgs = pkgs + extra_pkgs
 
         # generate a dependency tree for all our packages
-        tree = self.generateDepTreeData(pkgs, 'build', more_meta=True)
+        tree = self.generatePkgDepTreeData(pkgs, 'build')
         bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.configuration.data)
 
     def buildWorldTargetList(self):
-- 
1.7.6

Re: [PATCH 0/1] Remove unused target tree data for Hob

[Richard Purdie]

On Thu, 2011-07-21 at 12:08 -0700, Joshua Lock wrote:
> Here's a copy of Liping's "Remove unused target tree data for Hob" with my
> changes made per my review and my SoB added.
> 
> The following changes since commit 09a9146262d58dfe4a2ea4270026b90ae33f6c91:
> 
>   parse/ConfHandler: Fix multiline variable corruption (2011-07-21 11:08:19 +0100)
> 
> are available in the git repository at:
>   git://github.com/incandescant/bitbake lke-speed
>   https://github.com/incandescant/bitbake/tree/lke-speed
> 
> Liping Ke (1):
>   Remove unused target tree data for Hob

Merged to master, thanks.

Richard