* [PATCH 1/1][Image Creator] Remove unused target tree data for Hob
@ 2011-07-21 5:50 ` Ke Liping
0 siblings, 0 replies; 11+ messages in thread
From: Ke Liping @ 2011-07-21 5:50 UTC (permalink / raw)
To: yocto, bitbake-devel
From: Liping Ke <liping.ke@intel.com>
Since Hob only needes package dependency information, we can
create a new version of package information retrieving methods,
remove task dependency information, so that we can greatly
reduce data loading time for Hob
Signed-off-by: Liping Ke <liping.ke@intel.com>
---
bitbake/lib/bb/cooker.py | 96 +++++++++++++++++++++++++++++++++++++---------
1 files changed, 77 insertions(+), 19 deletions(-)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 8badd2d..26ed2ae 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -296,13 +296,12 @@ class BBCooker:
if data.getVarFlag( e, 'python', envdata ):
logger.plain("\npython %s () {\n%s}\n", e, data.getVar(e, envdata, 1))
- def prepareTreeData(self, pkgs_to_build, task):
+ def prepareTreeData(self, pkgs_to_build, task, runqueue=True):
"""
Prepare a runqueue and taskdata object for iteration over pkgs_to_build
"""
# Need files parsed
self.updateCache()
-
# If we are told to do the None task then query the default task
if (task == None):
task = self.configuration.cmd
@@ -322,12 +321,22 @@ class BBCooker:
runlist.append([k, "do_%s" % task])
taskdata.add_unresolved(localdata, self.status)
- rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
- rq.rqdata.prepare()
-
- return taskdata, rq
-
- def generateDepTreeData(self, pkgs_to_build, task, more_meta=False):
+ # For some users, take Hob as example, we only need less dependency data
+ # mark this kind of user as "less_meta" data requesterG
+ if (runqueue):
+ rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
+ rq.rqdata.prepare()
+ return taskdata, rq
+ else:
+ tasks_fnid = []
+ if len(taskdata.tasks_name) == 0:
+ # Nothing to do
+ return
+ for task in xrange(len(taskdata.tasks_name)):
+ tasks_fnid.append(taskdata.tasks_fnid[task])
+ return taskdata, tasks_fnid
+
+ def generateTaskDepTreeData(self, pkgs_to_build, task):
"""
Create a dependency tree of pkgs_to_build, returning the data.
When more_meta is set to True include summary, license and group
@@ -351,18 +360,10 @@ class BBCooker:
fn = taskdata.fn_index[fnid]
pn = self.status.pkg_fn[fn]
version = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
- if more_meta:
- summary = self.status.summary[fn]
- lic = self.status.license[fn]
- section = self.status.section[fn]
if pn not in depend_tree["pn"]:
depend_tree["pn"][pn] = {}
depend_tree["pn"][pn]["filename"] = fn
depend_tree["pn"][pn]["version"] = version
- if more_meta:
- depend_tree["pn"][pn]["summary"] = summary
- depend_tree["pn"][pn]["license"] = lic
- depend_tree["pn"][pn]["section"] = section
for dep in rq.rqdata.runq_depends[task]:
depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
deppn = self.status.pkg_fn[depfn]
@@ -407,12 +408,69 @@ class BBCooker:
return depend_tree
+ def generatePkgDepTreeData(self, pkgs_to_build, task):
+ """
+ Create a dependency tree of pkgs_to_build, returning the data.
+ When more_meta is set to True include summary, license and group
+ information in the returned tree.
+ """
+ taskdata, tasks_fnid = self.prepareTreeData(pkgs_to_build, task, False)
+
+ seen_fnids = []
+ depend_tree = {}
+ depend_tree["depends"] = {}
+ depend_tree["pn"] = {}
+ depend_tree["rdepends-pn"] = {}
+ depend_tree["packages"] = {}
+ depend_tree["rdepends-pkg"] = {}
+
+ for task in xrange(len(tasks_fnid)):
+ fnid = tasks_fnid[task]
+ fn = taskdata.fn_index[fnid]
+ pn = self.status.pkg_fn[fn]
+ version = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
+ summary = self.status.summary[fn]
+ lic = self.status.license[fn]
+ section = self.status.section[fn]
+ if pn not in depend_tree["pn"]:
+ depend_tree["pn"][pn] = {}
+ depend_tree["pn"][pn]["filename"] = fn
+ depend_tree["pn"][pn]["version"] = version
+ depend_tree["pn"][pn]["summary"] = summary
+ depend_tree["pn"][pn]["license"] = lic
+ depend_tree["pn"][pn]["section"] = section
+
+ if fnid not in seen_fnids:
+ seen_fnids.append(fnid)
+ packages = []
+
+ depend_tree["depends"][pn] = []
+ for dep in taskdata.depids[fnid]:
+ depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
+
+ depend_tree["rdepends-pn"][pn] = []
+ for rdep in taskdata.rdepids[fnid]:
+ depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
+
+ rdepends = self.status.rundeps[fn]
+ for package in rdepends:
+ packages.append(package)
+
+ for package in packages:
+ if package not in depend_tree["packages"]:
+ depend_tree["packages"][package] = {}
+ depend_tree["packages"][package]["pn"] = pn
+ depend_tree["packages"][package]["filename"] = fn
+ depend_tree["packages"][package]["version"] = version
+
+ return depend_tree
+
def generateDepTreeEvent(self, pkgs_to_build, task):
"""
Create a task dependency graph of pkgs_to_build.
Generate an event with the result
"""
- depgraph = self.generateDepTreeData(pkgs_to_build, task)
+ depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.configuration.data)
def generateDotGraphFiles(self, pkgs_to_build, task):
@@ -421,7 +479,7 @@ class BBCooker:
Save the result to a set of .dot files.
"""
- depgraph = self.generateDepTreeData(pkgs_to_build, task)
+ depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
# Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
depends_file = file('pn-depends.dot', 'w' )
@@ -628,7 +686,7 @@ class BBCooker:
pkgs = pkgs + extra_pkgs
# generate a dependency tree for all our packages
- tree = self.generateDepTreeData(pkgs, 'build', more_meta=True)
+ tree = self.generatePkgDepTreeData(pkgs, 'build')
bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.configuration.data)
def buildWorldTargetList(self):
--
1.7.0.4
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [PATCH 1/1][Image Creator] Remove unused target tree data for Hob
2011-07-21 5:50 ` [PATCH 1/1][Image Creator] " Ke Liping
(?)
@ 2011-07-20 0:39 ` Joshua Lock
2011-07-21 5:31 ` Ke, Liping
-1 siblings, 1 reply; 11+ messages in thread
From: Joshua Lock @ 2011-07-20 0:39 UTC (permalink / raw)
To: yocto
On Fri, 2011-07-15 at 11:31 +0800, Ke Liping wrote:
> From: Liping Ke <liping.ke@intel.com>
>
> Since Hob only needes package dependency information, we can
> create a new version of package information retrieving methods,
> remove task dependency information, so that we can greatly
> reduce data loading time for Hob
>
> Signed-off-by: Liping Ke <liping.ke@intel.com>
> ---
> bitbake/lib/bb/cooker.py | 96 +++++++++++++++++++++++++++++++++++++---------
> 1 files changed, 77 insertions(+), 19 deletions(-)
>
> diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
> index 8badd2d..26ed2ae 100644
> --- a/bitbake/lib/bb/cooker.py
> +++ b/bitbake/lib/bb/cooker.py
> @@ -296,13 +296,12 @@ class BBCooker:
> if data.getVarFlag( e, 'python', envdata ):
> logger.plain("\npython %s () {\n%s}\n", e, data.getVar(e, envdata, 1))
>
> - def prepareTreeData(self, pkgs_to_build, task):
> + def prepareTreeData(self, pkgs_to_build, task, runqueue=True):
> """
> Prepare a runqueue and taskdata object for iteration over pkgs_to_build
> """
> # Need files parsed
> self.updateCache()
> -
> # If we are told to do the None task then query the default task
> if (task == None):
> task = self.configuration.cmd
> @@ -322,12 +321,22 @@ class BBCooker:
> runlist.append([k, "do_%s" % task])
> taskdata.add_unresolved(localdata, self.status)
I'd prefer this method end around, not take the runqueue parameter and
just be responsible for updating the cache and returning taskdata
>
> - rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
> - rq.rqdata.prepare()
> -
> - return taskdata, rq
> -
> - def generateDepTreeData(self, pkgs_to_build, task, more_meta=False):
> + # For some users, take Hob as example, we only need less dependency data
> + # mark this kind of user as "less_meta" data requester
> + if (runqueue):
> + rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
> + rq.rqdata.prepare()
> + return taskdata, rq
Runqueue construction could then be rolled into generateTaskDepTreeData
> + else:
> + tasks_fnid = []
> + if len(taskdata.tasks_name) == 0:
> + # Nothing to do
> + return
> + for task in xrange(len(taskdata.tasks_name)):
> + tasks_fnid.append(taskdata.tasks_fnid[task])
> + return taskdata, tasks_fnid
and creation of tasks_fnid could be rolled into generatePkgDepTreeData
> +
> + def generateTaskDepTreeData(self, pkgs_to_build, task):
> """
> Create a dependency tree of pkgs_to_build, returning the data.
> When more_meta is set to True include summary, license and group
> @@ -351,18 +360,10 @@ class BBCooker:
> fn = taskdata.fn_index[fnid]
> pn = self.status.pkg_fn[fn]
> version = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
> - if more_meta:
> - summary = self.status.summary[fn]
> - lic = self.status.license[fn]
> - section = self.status.section[fn]
> if pn not in depend_tree["pn"]:
> depend_tree["pn"][pn] = {}
> depend_tree["pn"][pn]["filename"] = fn
> depend_tree["pn"][pn]["version"] = version
> - if more_meta:
> - depend_tree["pn"][pn]["summary"] = summary
> - depend_tree["pn"][pn]["license"] = lic
> - depend_tree["pn"][pn]["section"] = section
Glad to see these go :-)
> for dep in rq.rqdata.runq_depends[task]:
> depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
> deppn = self.status.pkg_fn[depfn]
> @@ -407,12 +408,69 @@ class BBCooker:
> return depend_tree
>
>
> + def generatePkgDepTreeData(self, pkgs_to_build, task):
> + """
> + Create a dependency tree of pkgs_to_build, returning the data.
> + When more_meta is set to True include summary, license and group
> + information in the returned tree.
> + """
> + taskdata, tasks_fnid = self.prepareTreeData(pkgs_to_build, task, False)
per above tasks_fnid would be populated here.
> +
> + seen_fnids = []
> + depend_tree = {}
> + depend_tree["depends"] = {}
> + depend_tree["pn"] = {}
> + depend_tree["rdepends-pn"] = {}
> + depend_tree["packages"] = {}
> + depend_tree["rdepends-pkg"] = {}
> +
> + for task in xrange(len(tasks_fnid)):
> + fnid = tasks_fnid[task]
> + fn = taskdata.fn_index[fnid]
> + pn = self.status.pkg_fn[fn]
> + version = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
> + summary = self.status.summary[fn]
> + lic = self.status.license[fn]
> + section = self.status.section[fn]
> + if pn not in depend_tree["pn"]:
> + depend_tree["pn"][pn] = {}
> + depend_tree["pn"][pn]["filename"] = fn
> + depend_tree["pn"][pn]["version"] = version
> + depend_tree["pn"][pn]["summary"] = summary
> + depend_tree["pn"][pn]["license"] = lic
> + depend_tree["pn"][pn]["section"] = section
> +
> + if fnid not in seen_fnids:
> + seen_fnids.append(fnid)
> + packages = []
> +
> + depend_tree["depends"][pn] = []
> + for dep in taskdata.depids[fnid]:
> + depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
> +
> + depend_tree["rdepends-pn"][pn] = []
> + for rdep in taskdata.rdepids[fnid]:
> + depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
> +
> + rdepends = self.status.rundeps[fn]
> + for package in rdepends:
> + packages.append(package)
> +
> + for package in packages:
> + if package not in depend_tree["packages"]:
> + depend_tree["packages"][package] = {}
> + depend_tree["packages"][package]["pn"] = pn
> + depend_tree["packages"][package]["filename"] = fn
> + depend_tree["packages"][package]["version"] = version
> +
> + return depend_tree
> +
> def generateDepTreeEvent(self, pkgs_to_build, task):
> """
> Create a task dependency graph of pkgs_to_build.
> Generate an event with the result
> """
> - depgraph = self.generateDepTreeData(pkgs_to_build, task)
> + depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
> bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.configuration.data)
>
> def generateDotGraphFiles(self, pkgs_to_build, task):
> @@ -421,7 +479,7 @@ class BBCooker:
> Save the result to a set of .dot files.
> """
>
> - depgraph = self.generateDepTreeData(pkgs_to_build, task)
> + depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
>
> # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
> depends_file = file('pn-depends.dot', 'w' )
> @@ -628,7 +686,7 @@ class BBCooker:
> pkgs = pkgs + extra_pkgs
>
> # generate a dependency tree for all our packages
> - tree = self.generateDepTreeData(pkgs, 'build', more_meta=True)
> + tree = self.generatePkgDepTreeData(pkgs, 'build')
> bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.configuration.data)
>
> def buildWorldTargetList(self):
Looking good to me. The revised patch would ideally be sent to
bitbake-devel@lists.openembedded.org
Thanks,
Joshua
--
Joshua Lock
Yocto Project "Johannes factotum"
Intel Open Source Technology Centre
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [yocto] [PATCH 1/1][Image Creator] Remove unused target tree data for Hob
2011-07-21 5:50 ` [PATCH 1/1][Image Creator] " Ke Liping
@ 2011-07-21 9:04 ` Koen Kooi
-1 siblings, 0 replies; 11+ messages in thread
From: Koen Kooi @ 2011-07-21 9:04 UTC (permalink / raw)
To: Ke Liping; +Cc: yocto, bitbake-devel
Op 21 jul. 2011, om 07:50 heeft Ke Liping het volgende geschreven:
> From: Liping Ke <liping.ke@intel.com>
>
> Since Hob only needes package dependency information, we can
> create a new version of package information retrieving methods,
> remove task dependency information, so that we can greatly
> reduce data loading time for Hob
Does this fix the bug where DEPENDS are leaking into the RDEPENDS list? If you select something like systemd the gui will list 'gtk' as a runtime dep for the image, while it's only a transitive buildtime one.
>
> Signed-off-by: Liping Ke <liping.ke@intel.com>
> ---
> bitbake/lib/bb/cooker.py | 96 +++++++++++++++++++++++++++++++++++++---------
> 1 files changed, 77 insertions(+), 19 deletions(-)
>
> diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
> index 8badd2d..26ed2ae 100644
> --- a/bitbake/lib/bb/cooker.py
> +++ b/bitbake/lib/bb/cooker.py
> @@ -296,13 +296,12 @@ class BBCooker:
> if data.getVarFlag( e, 'python', envdata ):
> logger.plain("\npython %s () {\n%s}\n", e, data.getVar(e, envdata, 1))
>
> - def prepareTreeData(self, pkgs_to_build, task):
> + def prepareTreeData(self, pkgs_to_build, task, runqueue=True):
> """
> Prepare a runqueue and taskdata object for iteration over pkgs_to_build
> """
> # Need files parsed
> self.updateCache()
> -
> # If we are told to do the None task then query the default task
> if (task == None):
> task = self.configuration.cmd
> @@ -322,12 +321,22 @@ class BBCooker:
> runlist.append([k, "do_%s" % task])
> taskdata.add_unresolved(localdata, self.status)
>
> - rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
> - rq.rqdata.prepare()
> -
> - return taskdata, rq
> -
> - def generateDepTreeData(self, pkgs_to_build, task, more_meta=False):
> + # For some users, take Hob as example, we only need less dependency data
> + # mark this kind of user as "less_meta" data requesterG
> + if (runqueue):
> + rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
> + rq.rqdata.prepare()
> + return taskdata, rq
> + else:
> + tasks_fnid = []
> + if len(taskdata.tasks_name) == 0:
> + # Nothing to do
> + return
> + for task in xrange(len(taskdata.tasks_name)):
> + tasks_fnid.append(taskdata.tasks_fnid[task])
> + return taskdata, tasks_fnid
> +
> + def generateTaskDepTreeData(self, pkgs_to_build, task):
> """
> Create a dependency tree of pkgs_to_build, returning the data.
> When more_meta is set to True include summary, license and group
> @@ -351,18 +360,10 @@ class BBCooker:
> fn = taskdata.fn_index[fnid]
> pn = self.status.pkg_fn[fn]
> version = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
> - if more_meta:
> - summary = self.status.summary[fn]
> - lic = self.status.license[fn]
> - section = self.status.section[fn]
> if pn not in depend_tree["pn"]:
> depend_tree["pn"][pn] = {}
> depend_tree["pn"][pn]["filename"] = fn
> depend_tree["pn"][pn]["version"] = version
> - if more_meta:
> - depend_tree["pn"][pn]["summary"] = summary
> - depend_tree["pn"][pn]["license"] = lic
> - depend_tree["pn"][pn]["section"] = section
> for dep in rq.rqdata.runq_depends[task]:
> depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
> deppn = self.status.pkg_fn[depfn]
> @@ -407,12 +408,69 @@ class BBCooker:
> return depend_tree
>
>
> + def generatePkgDepTreeData(self, pkgs_to_build, task):
> + """
> + Create a dependency tree of pkgs_to_build, returning the data.
> + When more_meta is set to True include summary, license and group
> + information in the returned tree.
> + """
> + taskdata, tasks_fnid = self.prepareTreeData(pkgs_to_build, task, False)
> +
> + seen_fnids = []
> + depend_tree = {}
> + depend_tree["depends"] = {}
> + depend_tree["pn"] = {}
> + depend_tree["rdepends-pn"] = {}
> + depend_tree["packages"] = {}
> + depend_tree["rdepends-pkg"] = {}
> +
> + for task in xrange(len(tasks_fnid)):
> + fnid = tasks_fnid[task]
> + fn = taskdata.fn_index[fnid]
> + pn = self.status.pkg_fn[fn]
> + version = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
> + summary = self.status.summary[fn]
> + lic = self.status.license[fn]
> + section = self.status.section[fn]
> + if pn not in depend_tree["pn"]:
> + depend_tree["pn"][pn] = {}
> + depend_tree["pn"][pn]["filename"] = fn
> + depend_tree["pn"][pn]["version"] = version
> + depend_tree["pn"][pn]["summary"] = summary
> + depend_tree["pn"][pn]["license"] = lic
> + depend_tree["pn"][pn]["section"] = section
> +
> + if fnid not in seen_fnids:
> + seen_fnids.append(fnid)
> + packages = []
> +
> + depend_tree["depends"][pn] = []
> + for dep in taskdata.depids[fnid]:
> + depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
> +
> + depend_tree["rdepends-pn"][pn] = []
> + for rdep in taskdata.rdepids[fnid]:
> + depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
> +
> + rdepends = self.status.rundeps[fn]
> + for package in rdepends:
> + packages.append(package)
> +
> + for package in packages:
> + if package not in depend_tree["packages"]:
> + depend_tree["packages"][package] = {}
> + depend_tree["packages"][package]["pn"] = pn
> + depend_tree["packages"][package]["filename"] = fn
> + depend_tree["packages"][package]["version"] = version
> +
> + return depend_tree
> +
> def generateDepTreeEvent(self, pkgs_to_build, task):
> """
> Create a task dependency graph of pkgs_to_build.
> Generate an event with the result
> """
> - depgraph = self.generateDepTreeData(pkgs_to_build, task)
> + depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
> bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.configuration.data)
>
> def generateDotGraphFiles(self, pkgs_to_build, task):
> @@ -421,7 +479,7 @@ class BBCooker:
> Save the result to a set of .dot files.
> """
>
> - depgraph = self.generateDepTreeData(pkgs_to_build, task)
> + depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
>
> # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
> depends_file = file('pn-depends.dot', 'w' )
> @@ -628,7 +686,7 @@ class BBCooker:
> pkgs = pkgs + extra_pkgs
>
> # generate a dependency tree for all our packages
> - tree = self.generateDepTreeData(pkgs, 'build', more_meta=True)
> + tree = self.generatePkgDepTreeData(pkgs, 'build')
> bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.configuration.data)
>
> def buildWorldTargetList(self):
> --
> 1.7.0.4
>
> _______________________________________________
> yocto mailing list
> yocto@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/yocto
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH 1/1][Image Creator] Remove unused target tree data for Hob
@ 2011-07-21 9:04 ` Koen Kooi
0 siblings, 0 replies; 11+ messages in thread
From: Koen Kooi @ 2011-07-21 9:04 UTC (permalink / raw)
To: Ke Liping; +Cc: yocto, bitbake-devel
Op 21 jul. 2011, om 07:50 heeft Ke Liping het volgende geschreven:
> From: Liping Ke <liping.ke@intel.com>
>
> Since Hob only needes package dependency information, we can
> create a new version of package information retrieving methods,
> remove task dependency information, so that we can greatly
> reduce data loading time for Hob
Does this fix the bug where DEPENDS are leaking into the RDEPENDS list? If you select something like systemd the gui will list 'gtk' as a runtime dep for the image, while it's only a transitive buildtime one.
>
> Signed-off-by: Liping Ke <liping.ke@intel.com>
> ---
> bitbake/lib/bb/cooker.py | 96 +++++++++++++++++++++++++++++++++++++---------
> 1 files changed, 77 insertions(+), 19 deletions(-)
>
> diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
> index 8badd2d..26ed2ae 100644
> --- a/bitbake/lib/bb/cooker.py
> +++ b/bitbake/lib/bb/cooker.py
> @@ -296,13 +296,12 @@ class BBCooker:
> if data.getVarFlag( e, 'python', envdata ):
> logger.plain("\npython %s () {\n%s}\n", e, data.getVar(e, envdata, 1))
>
> - def prepareTreeData(self, pkgs_to_build, task):
> + def prepareTreeData(self, pkgs_to_build, task, runqueue=True):
> """
> Prepare a runqueue and taskdata object for iteration over pkgs_to_build
> """
> # Need files parsed
> self.updateCache()
> -
> # If we are told to do the None task then query the default task
> if (task == None):
> task = self.configuration.cmd
> @@ -322,12 +321,22 @@ class BBCooker:
> runlist.append([k, "do_%s" % task])
> taskdata.add_unresolved(localdata, self.status)
>
> - rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
> - rq.rqdata.prepare()
> -
> - return taskdata, rq
> -
> - def generateDepTreeData(self, pkgs_to_build, task, more_meta=False):
> + # For some users, take Hob as example, we only need less dependency data
> + # mark this kind of user as "less_meta" data requesterG
> + if (runqueue):
> + rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
> + rq.rqdata.prepare()
> + return taskdata, rq
> + else:
> + tasks_fnid = []
> + if len(taskdata.tasks_name) == 0:
> + # Nothing to do
> + return
> + for task in xrange(len(taskdata.tasks_name)):
> + tasks_fnid.append(taskdata.tasks_fnid[task])
> + return taskdata, tasks_fnid
> +
> + def generateTaskDepTreeData(self, pkgs_to_build, task):
> """
> Create a dependency tree of pkgs_to_build, returning the data.
> When more_meta is set to True include summary, license and group
> @@ -351,18 +360,10 @@ class BBCooker:
> fn = taskdata.fn_index[fnid]
> pn = self.status.pkg_fn[fn]
> version = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
> - if more_meta:
> - summary = self.status.summary[fn]
> - lic = self.status.license[fn]
> - section = self.status.section[fn]
> if pn not in depend_tree["pn"]:
> depend_tree["pn"][pn] = {}
> depend_tree["pn"][pn]["filename"] = fn
> depend_tree["pn"][pn]["version"] = version
> - if more_meta:
> - depend_tree["pn"][pn]["summary"] = summary
> - depend_tree["pn"][pn]["license"] = lic
> - depend_tree["pn"][pn]["section"] = section
> for dep in rq.rqdata.runq_depends[task]:
> depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
> deppn = self.status.pkg_fn[depfn]
> @@ -407,12 +408,69 @@ class BBCooker:
> return depend_tree
>
>
> + def generatePkgDepTreeData(self, pkgs_to_build, task):
> + """
> + Create a dependency tree of pkgs_to_build, returning the data.
> + When more_meta is set to True include summary, license and group
> + information in the returned tree.
> + """
> + taskdata, tasks_fnid = self.prepareTreeData(pkgs_to_build, task, False)
> +
> + seen_fnids = []
> + depend_tree = {}
> + depend_tree["depends"] = {}
> + depend_tree["pn"] = {}
> + depend_tree["rdepends-pn"] = {}
> + depend_tree["packages"] = {}
> + depend_tree["rdepends-pkg"] = {}
> +
> + for task in xrange(len(tasks_fnid)):
> + fnid = tasks_fnid[task]
> + fn = taskdata.fn_index[fnid]
> + pn = self.status.pkg_fn[fn]
> + version = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
> + summary = self.status.summary[fn]
> + lic = self.status.license[fn]
> + section = self.status.section[fn]
> + if pn not in depend_tree["pn"]:
> + depend_tree["pn"][pn] = {}
> + depend_tree["pn"][pn]["filename"] = fn
> + depend_tree["pn"][pn]["version"] = version
> + depend_tree["pn"][pn]["summary"] = summary
> + depend_tree["pn"][pn]["license"] = lic
> + depend_tree["pn"][pn]["section"] = section
> +
> + if fnid not in seen_fnids:
> + seen_fnids.append(fnid)
> + packages = []
> +
> + depend_tree["depends"][pn] = []
> + for dep in taskdata.depids[fnid]:
> + depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
> +
> + depend_tree["rdepends-pn"][pn] = []
> + for rdep in taskdata.rdepids[fnid]:
> + depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
> +
> + rdepends = self.status.rundeps[fn]
> + for package in rdepends:
> + packages.append(package)
> +
> + for package in packages:
> + if package not in depend_tree["packages"]:
> + depend_tree["packages"][package] = {}
> + depend_tree["packages"][package]["pn"] = pn
> + depend_tree["packages"][package]["filename"] = fn
> + depend_tree["packages"][package]["version"] = version
> +
> + return depend_tree
> +
> def generateDepTreeEvent(self, pkgs_to_build, task):
> """
> Create a task dependency graph of pkgs_to_build.
> Generate an event with the result
> """
> - depgraph = self.generateDepTreeData(pkgs_to_build, task)
> + depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
> bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.configuration.data)
>
> def generateDotGraphFiles(self, pkgs_to_build, task):
> @@ -421,7 +479,7 @@ class BBCooker:
> Save the result to a set of .dot files.
> """
>
> - depgraph = self.generateDepTreeData(pkgs_to_build, task)
> + depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
>
> # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
> depends_file = file('pn-depends.dot', 'w' )
> @@ -628,7 +686,7 @@ class BBCooker:
> pkgs = pkgs + extra_pkgs
>
> # generate a dependency tree for all our packages
> - tree = self.generateDepTreeData(pkgs, 'build', more_meta=True)
> + tree = self.generatePkgDepTreeData(pkgs, 'build')
> bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.configuration.data)
>
> def buildWorldTargetList(self):
> --
> 1.7.0.4
>
> _______________________________________________
> yocto mailing list
> yocto@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/yocto
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [yocto] [PATCH 1/1][Image Creator] Remove unused target tree data for Hob
2011-07-21 9:04 ` Koen Kooi
@ 2011-07-21 18:38 ` Joshua Lock
-1 siblings, 0 replies; 11+ messages in thread
From: Joshua Lock @ 2011-07-21 18:38 UTC (permalink / raw)
To: Koen Kooi; +Cc: yocto, bitbake-devel
On Thu, 2011-07-21 at 11:04 +0200, Koen Kooi wrote:
> Op 21 jul. 2011, om 07:50 heeft Ke Liping het volgende geschreven:
>
> > From: Liping Ke <liping.ke@intel.com>
> >
> > Since Hob only needes package dependency information, we can
> > create a new version of package information retrieving methods,
> > remove task dependency information, so that we can greatly
> > reduce data loading time for Hob
>
>
> Does this fix the bug where DEPENDS are leaking into the RDEPENDS list?
> If you select something like systemd the gui will list 'gtk' as a
> runtime dep for the image, while it's only a transitive buildtime one.
>
It does not, 1263 filed.
http://bugzilla.pokylinux.org/show_bug.cgi?id=1263
Regards,
Joshua
--
Joshua Lock
Yocto Project "Johannes Factotum"
Intel Open Source Technology Centre
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [bitbake-devel] [PATCH 1/1][Image Creator] Remove unused target tree data for Hob
@ 2011-07-21 18:38 ` Joshua Lock
0 siblings, 0 replies; 11+ messages in thread
From: Joshua Lock @ 2011-07-21 18:38 UTC (permalink / raw)
To: Koen Kooi; +Cc: yocto, bitbake-devel
On Thu, 2011-07-21 at 11:04 +0200, Koen Kooi wrote:
> Op 21 jul. 2011, om 07:50 heeft Ke Liping het volgende geschreven:
>
> > From: Liping Ke <liping.ke@intel.com>
> >
> > Since Hob only needes package dependency information, we can
> > create a new version of package information retrieving methods,
> > remove task dependency information, so that we can greatly
> > reduce data loading time for Hob
>
>
> Does this fix the bug where DEPENDS are leaking into the RDEPENDS list?
> If you select something like systemd the gui will list 'gtk' as a
> runtime dep for the image, while it's only a transitive buildtime one.
>
It does not, 1263 filed.
http://bugzilla.pokylinux.org/show_bug.cgi?id=1263
Regards,
Joshua
--
Joshua Lock
Yocto Project "Johannes Factotum"
Intel Open Source Technology Centre
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH 1/1][Image Creator] Remove unused target tree data for Hob
2011-07-21 5:50 ` [PATCH 1/1][Image Creator] " Ke Liping
` (2 preceding siblings ...)
(?)
@ 2011-07-21 19:03 ` Joshua Lock
-1 siblings, 0 replies; 11+ messages in thread
From: Joshua Lock @ 2011-07-21 19:03 UTC (permalink / raw)
To: bitbake-devel
On Thu, 2011-07-21 at 13:50 +0800, Ke Liping wrote:
> From: Liping Ke <liping.ke@intel.com>
>
> Since Hob only needes package dependency information, we can
> create a new version of package information retrieving methods,
> remove task dependency information, so that we can greatly
> reduce data loading time for Hob
Liping, this patch looks reasonable and works as advertised however as
alluded to when I reviewed this patch on the yocto mailing list I don't
like the idea of a functions return type depending on an optional
parameter.
Therefore I've taken the liberty of changing the patch myself to prevent
you having to interpret my review and submit again. I hope you don't
mind?
I'm going to send another pull request with my modified patch and my
SoB.
Cheers,
Joshua
>
> Signed-off-by: Liping Ke <liping.ke@intel.com>
> ---
> bitbake/lib/bb/cooker.py | 96 +++++++++++++++++++++++++++++++++++++---------
> 1 files changed, 77 insertions(+), 19 deletions(-)
>
> diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
> index 8badd2d..26ed2ae 100644
> --- a/bitbake/lib/bb/cooker.py
> +++ b/bitbake/lib/bb/cooker.py
> @@ -296,13 +296,12 @@ class BBCooker:
> if data.getVarFlag( e, 'python', envdata ):
> logger.plain("\npython %s () {\n%s}\n", e, data.getVar(e, envdata, 1))
>
> - def prepareTreeData(self, pkgs_to_build, task):
> + def prepareTreeData(self, pkgs_to_build, task, runqueue=True):
See above.
> """
> Prepare a runqueue and taskdata object for iteration over pkgs_to_build
> """
> # Need files parsed
> self.updateCache()
> -
> # If we are told to do the None task then query the default task
> if (task == None):
> task = self.configuration.cmd
> @@ -322,12 +321,22 @@ class BBCooker:
> runlist.append([k, "do_%s" % task])
> taskdata.add_unresolved(localdata, self.status)
>
> - rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
> - rq.rqdata.prepare()
> -
> - return taskdata, rq
> -
> - def generateDepTreeData(self, pkgs_to_build, task, more_meta=False):
> + # For some users, take Hob as example, we only need less dependency data
> + # mark this kind of user as "less_meta" data requesterG
This comment is now wrong...
> + if (runqueue):
> + rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
> + rq.rqdata.prepare()
> + return taskdata, rq
> + else:
> + tasks_fnid = []
> + if len(taskdata.tasks_name) == 0:
> + # Nothing to do
> + return
> + for task in xrange(len(taskdata.tasks_name)):
> + tasks_fnid.append(taskdata.tasks_fnid[task])
> + return taskdata, tasks_fnid
> +
> + def generateTaskDepTreeData(self, pkgs_to_build, task):
> """
> Create a dependency tree of pkgs_to_build, returning the data.
> When more_meta is set to True include summary, license and group
This comment is also wrong
> @@ -351,18 +360,10 @@ class BBCooker:
> fn = taskdata.fn_index[fnid]
> pn = self.status.pkg_fn[fn]
> version = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
> - if more_meta:
> - summary = self.status.summary[fn]
> - lic = self.status.license[fn]
> - section = self.status.section[fn]
> if pn not in depend_tree["pn"]:
> depend_tree["pn"][pn] = {}
> depend_tree["pn"][pn]["filename"] = fn
> depend_tree["pn"][pn]["version"] = version
> - if more_meta:
> - depend_tree["pn"][pn]["summary"] = summary
> - depend_tree["pn"][pn]["license"] = lic
> - depend_tree["pn"][pn]["section"] = section
> for dep in rq.rqdata.runq_depends[task]:
> depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
> deppn = self.status.pkg_fn[depfn]
> @@ -407,12 +408,69 @@ class BBCooker:
> return depend_tree
>
>
> + def generatePkgDepTreeData(self, pkgs_to_build, task):
> + """
> + Create a dependency tree of pkgs_to_build, returning the data.
> + When more_meta is set to True include summary, license and group
> + information in the returned tree.
> + """
> + taskdata, tasks_fnid = self.prepareTreeData(pkgs_to_build, task, False)
> +
> + seen_fnids = []
> + depend_tree = {}
> + depend_tree["depends"] = {}
> + depend_tree["pn"] = {}
> + depend_tree["rdepends-pn"] = {}
> + depend_tree["packages"] = {}
> + depend_tree["rdepends-pkg"] = {}
> +
> + for task in xrange(len(tasks_fnid)):
> + fnid = tasks_fnid[task]
> + fn = taskdata.fn_index[fnid]
> + pn = self.status.pkg_fn[fn]
> + version = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
> + summary = self.status.summary[fn]
> + lic = self.status.license[fn]
> + section = self.status.section[fn]
> + if pn not in depend_tree["pn"]:
> + depend_tree["pn"][pn] = {}
> + depend_tree["pn"][pn]["filename"] = fn
> + depend_tree["pn"][pn]["version"] = version
> + depend_tree["pn"][pn]["summary"] = summary
> + depend_tree["pn"][pn]["license"] = lic
> + depend_tree["pn"][pn]["section"] = section
> +
> + if fnid not in seen_fnids:
> + seen_fnids.append(fnid)
> + packages = []
> +
> + depend_tree["depends"][pn] = []
> + for dep in taskdata.depids[fnid]:
> + depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
> +
> + depend_tree["rdepends-pn"][pn] = []
> + for rdep in taskdata.rdepids[fnid]:
> + depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
> +
> + rdepends = self.status.rundeps[fn]
> + for package in rdepends:
> + packages.append(package)
> +
> + for package in packages:
> + if package not in depend_tree["packages"]:
> + depend_tree["packages"][package] = {}
> + depend_tree["packages"][package]["pn"] = pn
> + depend_tree["packages"][package]["filename"] = fn
> + depend_tree["packages"][package]["version"] = version
> +
> + return depend_tree
> +
> def generateDepTreeEvent(self, pkgs_to_build, task):
> """
> Create a task dependency graph of pkgs_to_build.
> Generate an event with the result
> """
> - depgraph = self.generateDepTreeData(pkgs_to_build, task)
> + depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
> bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.configuration.data)
>
> def generateDotGraphFiles(self, pkgs_to_build, task):
> @@ -421,7 +479,7 @@ class BBCooker:
> Save the result to a set of .dot files.
> """
>
> - depgraph = self.generateDepTreeData(pkgs_to_build, task)
> + depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
>
> # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
> depends_file = file('pn-depends.dot', 'w' )
> @@ -628,7 +686,7 @@ class BBCooker:
> pkgs = pkgs + extra_pkgs
>
> # generate a dependency tree for all our packages
> - tree = self.generateDepTreeData(pkgs, 'build', more_meta=True)
> + tree = self.generatePkgDepTreeData(pkgs, 'build')
> bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.configuration.data)
>
> def buildWorldTargetList(self):
--
Joshua Lock
Yocto Project "Johannes Factotum"
Intel Open Source Technology Centre
^ permalink raw reply [flat|nested] 11+ messages in thread