* [PATCH 1/7] bitbake: cooker: clean up code and avoid duplication
2013-09-16 13:32 [PATCH v3 0/7] webhob related changes in bitbake, version 3 Alex DAMIAN
@ 2013-09-16 13:33 ` Alex DAMIAN
2013-09-16 13:33 ` [PATCH 2/7] bitbake: event: send the task dependency tree to UI Alex DAMIAN
` (5 subsequent siblings)
6 siblings, 0 replies; 17+ messages in thread
From: Alex DAMIAN @ 2013-09-16 13:33 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Clean-up to avoid duplication and promote code reuse.
We're using the prepareTreeData in all instances where
this needs to be done.
bb.event.BuildStarted now contains unexpanded package names,
since the UI will show the original target names.
When we're only testing target image contents, we do not abort
with an error if the target is unbuildable.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/cooker.py | 38 ++++++++++----------------------------
1 file changed, 10 insertions(+), 28 deletions(-)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index a07615b..3bfeefb 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -345,13 +345,7 @@ class BBCooker:
if pkgs_to_build[0] in set(ignore.split()):
bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
- localdata = data.createCopy(self.data)
- bb.data.update_data(localdata)
- bb.data.expandKeys(localdata)
-
- taskdata = bb.taskdata.TaskData(self.configuration.abort)
- taskdata.add_provider(localdata, self.recipecache, pkgs_to_build[0])
- taskdata.add_unresolved(localdata, self.recipecache)
+ runlist, taskdata = self.prepareTreeData(pkgs_to_build, None)
targetid = taskdata.getbuild_id(pkgs_to_build[0])
fnid = taskdata.build_targets[targetid][0]
@@ -398,9 +392,7 @@ class BBCooker:
localdata = data.createCopy(self.data)
bb.data.update_data(localdata)
bb.data.expandKeys(localdata)
- # We set abort to False here to prevent unbuildable targets raising
- # an exception when we're just generating data
- taskdata = bb.taskdata.TaskData(False, skiplist=self.skiplist)
+ taskdata = bb.taskdata.TaskData(self.configuration.abort, skiplist=self.skiplist)
runlist = []
current = 0
@@ -492,7 +484,12 @@ class BBCooker:
"""
Create a dependency tree of pkgs_to_build, returning the data.
"""
+
+ # We set abort to False here to prevent unbuildable targets raising
+ # an exception when we're just generating data
+ self.configuration.abort = False
_, taskdata = self.prepareTreeData(pkgs_to_build, task)
+
tasks_fnid = []
if len(taskdata.tasks_name) != 0:
for task in xrange(len(taskdata.tasks_name)):
@@ -1073,13 +1070,6 @@ class BBCooker:
Attempt to build the targets specified
"""
- # If we are told to do the NULL task then query the default task
- if (task == None):
- task = self.configuration.cmd
-
- universe = ('universe' in targets)
- targets = self.checkPackages(targets)
-
def buildTargetsIdle(server, rq, abort):
if abort or self.state == state.forceshutdown:
rq.finish_runqueue(True)
@@ -1106,21 +1096,13 @@ class BBCooker:
self.buildSetVars()
buildname = self.data.getVar("BUILDNAME")
- bb.event.fire(bb.event.BuildStarted(buildname, targets), self.data)
-
- localdata = data.createCopy(self.data)
- bb.data.update_data(localdata)
- bb.data.expandKeys(localdata)
- taskdata = bb.taskdata.TaskData(self.configuration.abort, skiplist=self.skiplist)
+ bb.event.fire(bb.event.BuildStarted(buildname, targets), self.data)
- runlist = []
- for k in targets:
- taskdata.add_provider(localdata, self.recipecache, k)
- runlist.append([k, "do_%s" % task])
- taskdata.add_unresolved(localdata, self.recipecache)
+ runlist, taskdata = self.prepareTreeData(targets, task)
rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist)
+ universe = ('universe' in targets)
if universe:
rq.rqdata.warn_multi_bb = True
--
1.8.1.2
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 2/7] bitbake: event: send the task dependency tree to UI
2013-09-16 13:32 [PATCH v3 0/7] webhob related changes in bitbake, version 3 Alex DAMIAN
2013-09-16 13:33 ` [PATCH 1/7] bitbake: cooker: clean up code and avoid duplication Alex DAMIAN
@ 2013-09-16 13:33 ` Alex DAMIAN
2013-09-16 15:32 ` Richard Purdie
2013-09-16 13:33 ` [PATCH 3/7] bitbake: runqueue: add runQueueTaskSkipped event Alex DAMIAN
` (4 subsequent siblings)
6 siblings, 1 reply; 17+ messages in thread
From: Alex DAMIAN @ 2013-09-16 13:33 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
We add an event that will send the dependency tree
graph to the UI client once it is computed.
This will allow the clients to display dependency
data in an efficient manner, and not redo the runqueue
computation specifically to get the dependency data.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/cooker.py | 3 +++
bitbake/lib/bb/runqueue.py | 3 +++
bitbake/lib/bb/ui/knotty.py | 3 +++
3 files changed, 9 insertions(+)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 3bfeefb..68a3c01 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -415,7 +415,10 @@ class BBCooker:
runlist, taskdata = self.prepareTreeData(pkgs_to_build, task)
rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist)
rq.rqdata.prepare()
+ return self.buildDependTree(rq, taskdata)
+
+ def buildDependTree(self, rq, taskdata):
seen_fnids = []
depend_tree = {}
depend_tree["depends"] = {}
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 25f1ab5..aa2f147 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -999,6 +999,9 @@ class RunQueue:
else:
self.state = runQueueSceneInit
+ depgraph = self.cooker.buildDependTree(self, self.rqdata.taskData)
+ bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data)
+
if self.state is runQueueSceneInit:
if self.cooker.configuration.dump_signatures:
self.dump_signatures()
diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index 35590a2..0211b50 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -472,6 +472,9 @@ def main(server, eventHandler, params, tf = TerminalFilter):
event.taskid, event.taskstring, event.exitcode)
continue
+ if isinstance(event, bb.event.DepTreeGenerated):
+ continue
+
# ignore
if isinstance(event, (bb.event.BuildBase,
bb.event.StampUpdate,
--
1.8.1.2
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [PATCH 2/7] bitbake: event: send the task dependency tree to UI
2013-09-16 13:33 ` [PATCH 2/7] bitbake: event: send the task dependency tree to UI Alex DAMIAN
@ 2013-09-16 15:32 ` Richard Purdie
0 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2013-09-16 15:32 UTC (permalink / raw)
To: Alex DAMIAN; +Cc: bitbake-devel
On Mon, 2013-09-16 at 14:33 +0100, Alex DAMIAN wrote:
> From: Alexandru DAMIAN <alexandru.damian@intel.com>
>
> We add an event that will send the dependency tree
> graph to the UI client once it is computed.
>
> This will allow the clients to display dependency
> data in an efficient manner, and not redo the runqueue
> computation specifically to get the dependency data.
>
> Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
> ---
> bitbake/lib/bb/cooker.py | 3 +++
> bitbake/lib/bb/runqueue.py | 3 +++
> bitbake/lib/bb/ui/knotty.py | 3 +++
> 3 files changed, 9 insertions(+)
I continue to feel quite strongly that this should only happen when
there are UIs connected that actually want this event. We actually have
two problems, this one and the part in bin/bitbake which says:
# Collect all the caches we need
if configParams.server_only:
configuration.extra_caches = gather_extra_cache_data()
else:
configuration.extra_caches = getattr(ui_module, "extraCaches", [])
I was never happy when this went in and I've already had one complaint
about the memory resident bitbake server reparsing compared to the
normal knotty client. You asked how this could be fixed as you couldn't
see how it could be done. I have two ideas:
The first is simpler but only fixes this problem, not the caches one.
The basic idea there would be to query the event interface and see if
anyone was listening for DepTreeGenerated events. If nobody is, no point
in generating it.
The second idea was having "server features", the caches being a special
case of this. When UI clients connect, they would merge their features
into those of the server. I've shown some pseudo code below. The
"features" could take immediate effect, the caches requested would take
effect the next time the server reparses (i.e finish current command,
then they take effect). The server features would be sent to the server
via some special command which would be allowed in an otherwise readonly
server.
Does that help?
Cheers,
Richard
--- a/bitbake/bin/bitbake
+++ b/bitbake/bin/bitbake
@@ -327,6 +327,10 @@ def main():
os.environ[k] = cleanedvars[k]
try:
+ if configParams.remote_server:
+ # Request configuration.extra_caches
+ # Request server feature flags set in UI
+
return ui_module.main(server_connection.connection, server_connection.events, configParams)
finally:
bb.event.ui_queue = []
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 2c8d4dc..89bf7c6 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -63,6 +63,20 @@ class CollectionError(bb.BBHandledException):
class state:
initial, parsing, running, shutdown, forceshutdown, stopped = range(6)
+#
+# Cooker Features control certain functionality in the server. This allows
+# things to happen only when we know we have a UI that needs the particular
+# feature. This is usually used to control event generation
+#
+class CookerFeatures(object):
+ def __init__(self):
+ self._features = ["generateRunQueueDepTreeEvents"]
+ for f in self._features:
+ setattr(self, f, False)
+ def merge(self, features):
+ for f in self._features:
+ v = getattr(features, f)
+ setattr(self, f, v)
class SkippedPackage:
def __init__(self, info = None, reason = None):
^ permalink raw reply related [flat|nested] 17+ messages in thread
* [PATCH 3/7] bitbake: runqueue: add runQueueTaskSkipped event
2013-09-16 13:32 [PATCH v3 0/7] webhob related changes in bitbake, version 3 Alex DAMIAN
2013-09-16 13:33 ` [PATCH 1/7] bitbake: cooker: clean up code and avoid duplication Alex DAMIAN
2013-09-16 13:33 ` [PATCH 2/7] bitbake: event: send the task dependency tree to UI Alex DAMIAN
@ 2013-09-16 13:33 ` Alex DAMIAN
2013-09-16 14:24 ` Richard Purdie
2013-09-16 13:33 ` [PATCH 4/7] bitbake: cooker: add extra recipe information Alex DAMIAN
` (3 subsequent siblings)
6 siblings, 1 reply; 17+ messages in thread
From: Alex DAMIAN @ 2013-09-16 13:33 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Adding a runQueueTaskSkipped to notify all listeners
about the tasks that are not run either because they
are set-scened or they don't need an update (timestamps
are ok)
Adds RunQueueData functions to get the task name and task
file for usage with the runQueue* events.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/build.py | 3 +++
bitbake/lib/bb/runqueue.py | 27 ++++++++++++++++++++++++---
2 files changed, 27 insertions(+), 3 deletions(-)
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index a53aba9..1265bdc 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -72,6 +72,9 @@ class TaskBase(event.Event):
def __init__(self, t, d ):
self._task = t
self._package = d.getVar("PF", True)
+ self._file = d.getVar("FILE", True)
+ self.taskfile = self._file
+ self.taskname = self._task
event.Event.__init__(self)
self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index aa2f147..2ab4405 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -217,6 +217,12 @@ class RunQueueData:
ret.extend([nam])
return ret
+ def get_task_name(self, task):
+ return self.runq_task[task]
+
+ def get_task_file(self, task):
+ return self.taskData.fn_index[self.runq_fnid[task]]
+
def get_user_idstring(self, task, task_name_suffix = ""):
fn = self.taskData.fn_index[self.runq_fnid[task]]
taskname = self.runq_task[task] + task_name_suffix
@@ -1328,9 +1334,10 @@ class RunQueueExecuteTasks(RunQueueExecute):
if self.rqdata.taskData.abort:
self.rq.state = runQueueCleanUp
- def task_skip(self, task):
+ def task_skip(self, task, skip_reason = ""):
self.runq_running[task] = 1
self.runq_buildable[task] = 1
+ bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, skip_reason), self.cfgData)
self.task_completeoutright(task)
self.stats.taskCompleted()
self.stats.taskSkipped()
@@ -1355,13 +1362,13 @@ class RunQueueExecuteTasks(RunQueueExecute):
if task in self.rq.scenequeue_covered:
logger.debug(2, "Setscene covered task %s (%s)", task,
self.rqdata.get_user_idstring(task))
- self.task_skip(task)
+ self.task_skip(task, "covered")
return True
if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
logger.debug(2, "Stamp current task %s (%s)", task,
self.rqdata.get_user_idstring(task))
- self.task_skip(task)
+ self.task_skip(task, "existing")
return True
taskdep = self.rqdata.dataCache.task_deps[fn]
@@ -1783,6 +1790,8 @@ class runQueueEvent(bb.event.Event):
def __init__(self, task, stats, rq):
self.taskid = task
self.taskstring = rq.rqdata.get_user_idstring(task)
+ self.taskname = rq.rqdata.get_task_name(task)
+ self.taskfile = rq.rqdata.get_task_file(task)
self.stats = stats.copy()
bb.event.Event.__init__(self)
@@ -1794,6 +1803,8 @@ class sceneQueueEvent(runQueueEvent):
runQueueEvent.__init__(self, task, stats, rq)
realtask = rq.rqdata.runq_setscene[task]
self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
+ self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene"
+ self.taskfile = rq.rqdata.get_task_file(realtask)
class runQueueTaskStarted(runQueueEvent):
"""
@@ -1837,6 +1848,16 @@ class sceneQueueTaskCompleted(sceneQueueEvent):
Event notifing a setscene task completed
"""
+class runQueueTaskSkipped(runQueueEvent):
+ """
+ Event notifing a task was skipped
+ """
+ def __init__(self, task, stats, rq, skip_reason):
+ runQueueEvent.__init__(self, task, stats, rq)
+ self._file = rq.rqdata.get_task_file(task)
+ self._task = rq.rqdata.get_task_name(task)
+ self._skip = skip_reason
+
class runQueuePipe():
"""
Abstraction for a pipe between a worker thread and the server
--
1.8.1.2
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [PATCH 3/7] bitbake: runqueue: add runQueueTaskSkipped event
2013-09-16 13:33 ` [PATCH 3/7] bitbake: runqueue: add runQueueTaskSkipped event Alex DAMIAN
@ 2013-09-16 14:24 ` Richard Purdie
2013-09-16 14:32 ` Damian, Alexandru
0 siblings, 1 reply; 17+ messages in thread
From: Richard Purdie @ 2013-09-16 14:24 UTC (permalink / raw)
To: Alex DAMIAN; +Cc: bitbake-devel
On Mon, 2013-09-16 at 14:33 +0100, Alex DAMIAN wrote:
> From: Alexandru DAMIAN <alexandru.damian@intel.com>
>
> Adding a runQueueTaskSkipped to notify all listeners
> about the tasks that are not run either because they
> are set-scened or they don't need an update (timestamps
> are ok)
>
> Adds RunQueueData functions to get the task name and task
> file for usage with the runQueue* events.
>
> Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
> ---
> bitbake/lib/bb/build.py | 3 +++
> bitbake/lib/bb/runqueue.py | 27 ++++++++++++++++++++++++---
> 2 files changed, 27 insertions(+), 3 deletions(-)
>
> diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
> index a53aba9..1265bdc 100644
> --- a/bitbake/lib/bb/build.py
> +++ b/bitbake/lib/bb/build.py
> @@ -72,6 +72,9 @@ class TaskBase(event.Event):
> def __init__(self, t, d ):
> self._task = t
> self._package = d.getVar("PF", True)
> + self._file = d.getVar("FILE", True)
> + self.taskfile = self._file
> + self.taskname = self._task
> event.Event.__init__(self)
> self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
>
> diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
> index aa2f147..2ab4405 100644
> --- a/bitbake/lib/bb/runqueue.py
> +++ b/bitbake/lib/bb/runqueue.py
> @@ -217,6 +217,12 @@ class RunQueueData:
> ret.extend([nam])
> return ret
>
> + def get_task_name(self, task):
> + return self.runq_task[task]
> +
> + def get_task_file(self, task):
> + return self.taskData.fn_index[self.runq_fnid[task]]
> +
> def get_user_idstring(self, task, task_name_suffix = ""):
> fn = self.taskData.fn_index[self.runq_fnid[task]]
> taskname = self.runq_task[task] + task_name_suffix
> @@ -1328,9 +1334,10 @@ class RunQueueExecuteTasks(RunQueueExecute):
> if self.rqdata.taskData.abort:
> self.rq.state = runQueueCleanUp
>
> - def task_skip(self, task):
> + def task_skip(self, task, skip_reason = ""):
> self.runq_running[task] = 1
> self.runq_buildable[task] = 1
> + bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, skip_reason), self.cfgData)
> self.task_completeoutright(task)
> self.stats.taskCompleted()
> self.stats.taskSkipped()
> @@ -1355,13 +1362,13 @@ class RunQueueExecuteTasks(RunQueueExecute):
> if task in self.rq.scenequeue_covered:
> logger.debug(2, "Setscene covered task %s (%s)", task,
> self.rqdata.get_user_idstring(task))
> - self.task_skip(task)
> + self.task_skip(task, "covered")
> return True
>
> if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
> logger.debug(2, "Stamp current task %s (%s)", task,
> self.rqdata.get_user_idstring(task))
> - self.task_skip(task)
> + self.task_skip(task, "existing")
> return True
>
> taskdep = self.rqdata.dataCache.task_deps[fn]
> @@ -1783,6 +1790,8 @@ class runQueueEvent(bb.event.Event):
> def __init__(self, task, stats, rq):
> self.taskid = task
> self.taskstring = rq.rqdata.get_user_idstring(task)
> + self.taskname = rq.rqdata.get_task_name(task)
> + self.taskfile = rq.rqdata.get_task_file(task)
> self.stats = stats.copy()
> bb.event.Event.__init__(self)
>
> @@ -1794,6 +1803,8 @@ class sceneQueueEvent(runQueueEvent):
> runQueueEvent.__init__(self, task, stats, rq)
> realtask = rq.rqdata.runq_setscene[task]
> self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
> + self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene"
> + self.taskfile = rq.rqdata.get_task_file(realtask)
>
> class runQueueTaskStarted(runQueueEvent):
> """
> @@ -1837,6 +1848,16 @@ class sceneQueueTaskCompleted(sceneQueueEvent):
> Event notifing a setscene task completed
> """
>
> +class runQueueTaskSkipped(runQueueEvent):
> + """
> + Event notifing a task was skipped
> + """
> + def __init__(self, task, stats, rq, skip_reason):
> + runQueueEvent.__init__(self, task, stats, rq)
> + self._file = rq.rqdata.get_task_file(task)
> + self._task = rq.rqdata.get_task_name(task)
> + self._skip = skip_reason
> +
> class runQueuePipe():
> """
> Abstraction for a pipe between a worker thread and the server
Please rebase this one off
http://git.yoctoproject.org/cgit.cgi/poky/commit/?h=master-next&id=39c70d0ad00afa05d343dc19aa557a092db7c963
since we should do one thing in a commit, not three. I renamed _skip ->
reason since its not internal and its a skip event, we know this, the
field is the reason.
You need to separate out the runQueueEvent changes from the TaskBase
changes.
Why add self._file to TaskBase? Surely just taskfile would be enough?
The _ is meant to indicate internal use only and aren't the above events
inconsistent with each other?
Cheers,
Richard
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 3/7] bitbake: runqueue: add runQueueTaskSkipped event
2013-09-16 14:24 ` Richard Purdie
@ 2013-09-16 14:32 ` Damian, Alexandru
2013-09-16 15:23 ` Richard Purdie
0 siblings, 1 reply; 17+ messages in thread
From: Damian, Alexandru @ 2013-09-16 14:32 UTC (permalink / raw)
To: Richard Purdie; +Cc: bitbake-devel
[-- Attachment #1: Type: text/plain, Size: 6047 bytes --]
On Mon, Sep 16, 2013 at 3:24 PM, Richard Purdie <
richard.purdie@linuxfoundation.org> wrote:
> On Mon, 2013-09-16 at 14:33 +0100, Alex DAMIAN wrote:
> > From: Alexandru DAMIAN <alexandru.damian@intel.com>
> >
> > Adding a runQueueTaskSkipped to notify all listeners
> > about the tasks that are not run either because they
> > are set-scened or they don't need an update (timestamps
> > are ok)
> >
> > Adds RunQueueData functions to get the task name and task
> > file for usage with the runQueue* events.
> >
> > Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
> > ---
> > bitbake/lib/bb/build.py | 3 +++
> > bitbake/lib/bb/runqueue.py | 27 ++++++++++++++++++++++++---
> > 2 files changed, 27 insertions(+), 3 deletions(-)
> >
> > diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
> > index a53aba9..1265bdc 100644
> > --- a/bitbake/lib/bb/build.py
> > +++ b/bitbake/lib/bb/build.py
> > @@ -72,6 +72,9 @@ class TaskBase(event.Event):
> > def __init__(self, t, d ):
> > self._task = t
> > self._package = d.getVar("PF", True)
> > + self._file = d.getVar("FILE", True)
> > + self.taskfile = self._file
> > + self.taskname = self._task
> > event.Event.__init__(self)
> > self._message = "recipe %s: task %s: %s" % (d.getVar("PF",
> True), t, self.getDisplayName())
> >
> > diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
> > index aa2f147..2ab4405 100644
> > --- a/bitbake/lib/bb/runqueue.py
> > +++ b/bitbake/lib/bb/runqueue.py
> > @@ -217,6 +217,12 @@ class RunQueueData:
> > ret.extend([nam])
> > return ret
> >
> > + def get_task_name(self, task):
> > + return self.runq_task[task]
> > +
> > + def get_task_file(self, task):
> > + return self.taskData.fn_index[self.runq_fnid[task]]
> > +
> > def get_user_idstring(self, task, task_name_suffix = ""):
> > fn = self.taskData.fn_index[self.runq_fnid[task]]
> > taskname = self.runq_task[task] + task_name_suffix
> > @@ -1328,9 +1334,10 @@ class RunQueueExecuteTasks(RunQueueExecute):
> > if self.rqdata.taskData.abort:
> > self.rq.state = runQueueCleanUp
> >
> > - def task_skip(self, task):
> > + def task_skip(self, task, skip_reason = ""):
> > self.runq_running[task] = 1
> > self.runq_buildable[task] = 1
> > + bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq,
> skip_reason), self.cfgData)
> > self.task_completeoutright(task)
> > self.stats.taskCompleted()
> > self.stats.taskSkipped()
> > @@ -1355,13 +1362,13 @@ class RunQueueExecuteTasks(RunQueueExecute):
> > if task in self.rq.scenequeue_covered:
> > logger.debug(2, "Setscene covered task %s (%s)", task,
> > self.rqdata.get_user_idstring(task))
> > - self.task_skip(task)
> > + self.task_skip(task, "covered")
> > return True
> >
> > if self.rq.check_stamp_task(task, taskname,
> cache=self.stampcache):
> > logger.debug(2, "Stamp current task %s (%s)", task,
> > self.rqdata.get_user_idstring(task))
> > - self.task_skip(task)
> > + self.task_skip(task, "existing")
> > return True
> >
> > taskdep = self.rqdata.dataCache.task_deps[fn]
> > @@ -1783,6 +1790,8 @@ class runQueueEvent(bb.event.Event):
> > def __init__(self, task, stats, rq):
> > self.taskid = task
> > self.taskstring = rq.rqdata.get_user_idstring(task)
> > + self.taskname = rq.rqdata.get_task_name(task)
> > + self.taskfile = rq.rqdata.get_task_file(task)
> > self.stats = stats.copy()
> > bb.event.Event.__init__(self)
> >
> > @@ -1794,6 +1803,8 @@ class sceneQueueEvent(runQueueEvent):
> > runQueueEvent.__init__(self, task, stats, rq)
> > realtask = rq.rqdata.runq_setscene[task]
> > self.taskstring = rq.rqdata.get_user_idstring(realtask,
> "_setscene")
> > + self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene"
> > + self.taskfile = rq.rqdata.get_task_file(realtask)
> >
> > class runQueueTaskStarted(runQueueEvent):
> > """
> > @@ -1837,6 +1848,16 @@ class sceneQueueTaskCompleted(sceneQueueEvent):
> > Event notifing a setscene task completed
> > """
> >
> > +class runQueueTaskSkipped(runQueueEvent):
> > + """
> > + Event notifing a task was skipped
> > + """
> > + def __init__(self, task, stats, rq, skip_reason):
> > + runQueueEvent.__init__(self, task, stats, rq)
> > + self._file = rq.rqdata.get_task_file(task)
> > + self._task = rq.rqdata.get_task_name(task)
> > + self._skip = skip_reason
> > +
> > class runQueuePipe():
> > """
> > Abstraction for a pipe between a worker thread and the server
>
>
> Please rebase this one off
>
> http://git.yoctoproject.org/cgit.cgi/poky/commit/?h=master-next&id=39c70d0ad00afa05d343dc19aa557a092db7c963
>
> since we should do one thing in a commit, not three. I renamed _skip ->
> reason since its not internal and its a skip event, we know this, the
> field is the reason.
>
> You need to separate out the runQueueEvent changes from the TaskBase
> changes.
>
> Why add self._file to TaskBase? Surely just taskfile would be enough?
> The _ is meant to indicate internal use only and aren't the above events
> inconsistent with each other?
>
[Alex] This is the idea - I actually tried to make TaskBase-derived events
consistent with runQueueEvents in terms of class fields, so the code that
reads them is easier to follow - I just have to look up for the same named
field regardless of the event type.
>
> Cheers,
>
> Richard
>
>
--
Alex Damian
Yocto Project
SSG / OTC
[-- Attachment #2: Type: text/html, Size: 8032 bytes --]
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 3/7] bitbake: runqueue: add runQueueTaskSkipped event
2013-09-16 14:32 ` Damian, Alexandru
@ 2013-09-16 15:23 ` Richard Purdie
0 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2013-09-16 15:23 UTC (permalink / raw)
To: Damian, Alexandru; +Cc: bitbake-devel
On Mon, 2013-09-16 at 15:32 +0100, Damian, Alexandru wrote:
>
>
>
> On Mon, Sep 16, 2013 at 3:24 PM, Richard Purdie
> <richard.purdie@linuxfoundation.org> wrote:
> On Mon, 2013-09-16 at 14:33 +0100, Alex DAMIAN wrote:
> > From: Alexandru DAMIAN <alexandru.damian@intel.com>
> >
> > Adding a runQueueTaskSkipped to notify all listeners
> > about the tasks that are not run either because they
> > are set-scened or they don't need an update (timestamps
> > are ok)
> >
> > Adds RunQueueData functions to get the task name and task
> > file for usage with the runQueue* events.
> >
> > Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
> > ---
> > bitbake/lib/bb/build.py | 3 +++
> > bitbake/lib/bb/runqueue.py | 27 ++++++++++++++++++++++++---
> > 2 files changed, 27 insertions(+), 3 deletions(-)
> >
> > diff --git a/bitbake/lib/bb/build.py
> b/bitbake/lib/bb/build.py
> > index a53aba9..1265bdc 100644
> > --- a/bitbake/lib/bb/build.py
> > +++ b/bitbake/lib/bb/build.py
> > @@ -72,6 +72,9 @@ class TaskBase(event.Event):
> > def __init__(self, t, d ):
> > self._task = t
> > self._package = d.getVar("PF", True)
> > + self._file = d.getVar("FILE", True)
> > + self.taskfile = self._file
> > + self.taskname = self._task
> > event.Event.__init__(self)
> > self._message = "recipe %s: task %s: %s" %
> (d.getVar("PF", True), t, self.getDisplayName())
> >
> > diff --git a/bitbake/lib/bb/runqueue.py
> b/bitbake/lib/bb/runqueue.py
> > index aa2f147..2ab4405 100644
> > --- a/bitbake/lib/bb/runqueue.py
> > +++ b/bitbake/lib/bb/runqueue.py
> > @@ -217,6 +217,12 @@ class RunQueueData:
> > ret.extend([nam])
> > return ret
> >
> > + def get_task_name(self, task):
> > + return self.runq_task[task]
> > +
> > + def get_task_file(self, task):
> > + return self.taskData.fn_index[self.runq_fnid[task]]
> > +
> > def get_user_idstring(self, task, task_name_suffix =
> ""):
> > fn = self.taskData.fn_index[self.runq_fnid[task]]
> > taskname = self.runq_task[task] + task_name_suffix
> > @@ -1328,9 +1334,10 @@ class
> RunQueueExecuteTasks(RunQueueExecute):
> > if self.rqdata.taskData.abort:
> > self.rq.state = runQueueCleanUp
> >
> > - def task_skip(self, task):
> > + def task_skip(self, task, skip_reason = ""):
> > self.runq_running[task] = 1
> > self.runq_buildable[task] = 1
> > + bb.event.fire(runQueueTaskSkipped(task, self.stats,
> self.rq, skip_reason), self.cfgData)
> > self.task_completeoutright(task)
> > self.stats.taskCompleted()
> > self.stats.taskSkipped()
> > @@ -1355,13 +1362,13 @@ class
> RunQueueExecuteTasks(RunQueueExecute):
> > if task in self.rq.scenequeue_covered:
> > logger.debug(2, "Setscene covered task %s
> (%s)", task,
> >
> self.rqdata.get_user_idstring(task))
> > - self.task_skip(task)
> > + self.task_skip(task, "covered")
> > return True
> >
> > if self.rq.check_stamp_task(task, taskname,
> cache=self.stampcache):
> > logger.debug(2, "Stamp current task %s (%
> s)", task,
> >
> self.rqdata.get_user_idstring(task))
> > - self.task_skip(task)
> > + self.task_skip(task, "existing")
> > return True
> >
> > taskdep = self.rqdata.dataCache.task_deps[fn]
> > @@ -1783,6 +1790,8 @@ class runQueueEvent(bb.event.Event):
> > def __init__(self, task, stats, rq):
> > self.taskid = task
> > self.taskstring = rq.rqdata.get_user_idstring(task)
> > + self.taskname = rq.rqdata.get_task_name(task)
> > + self.taskfile = rq.rqdata.get_task_file(task)
> > self.stats = stats.copy()
> > bb.event.Event.__init__(self)
> >
> > @@ -1794,6 +1803,8 @@ class sceneQueueEvent(runQueueEvent):
> > runQueueEvent.__init__(self, task, stats, rq)
> > realtask = rq.rqdata.runq_setscene[task]
> > self.taskstring =
> rq.rqdata.get_user_idstring(realtask, "_setscene")
> > + self.taskname = rq.rqdata.get_task_name(realtask) +
> "_setscene"
> > + self.taskfile = rq.rqdata.get_task_file(realtask)
> >
> > class runQueueTaskStarted(runQueueEvent):
> > """
> > @@ -1837,6 +1848,16 @@ class
> sceneQueueTaskCompleted(sceneQueueEvent):
> > Event notifing a setscene task completed
> > """
> >
> > +class runQueueTaskSkipped(runQueueEvent):
> > + """
> > + Event notifing a task was skipped
> > + """
> > + def __init__(self, task, stats, rq, skip_reason):
> > + runQueueEvent.__init__(self, task, stats, rq)
> > + self._file = rq.rqdata.get_task_file(task)
> > + self._task = rq.rqdata.get_task_name(task)
> > + self._skip = skip_reason
> > +
> > class runQueuePipe():
> > """
> > Abstraction for a pipe between a worker thread and the
> server
>
>
>
> Please rebase this one off
> http://git.yoctoproject.org/cgit.cgi/poky/commit/?h=master-next&id=39c70d0ad00afa05d343dc19aa557a092db7c963
>
> since we should do one thing in a commit, not three. I renamed
> _skip ->
> reason since its not internal and its a skip event, we know
> this, the
> field is the reason.
>
> You need to separate out the runQueueEvent changes from the
> TaskBase
> changes.
>
> Why add self._file to TaskBase? Surely just taskfile would be
> enough?
> The _ is meant to indicate internal use only and aren't the
> above events
> inconsistent with each other?
>
> [Alex] This is the idea - I actually tried to make TaskBase-derived
> events consistent with runQueueEvents in terms of class fields, so the
> code that reads them is easier to follow - I just have to look up for
> the same named field regardless of the event type.
>
No events have a _file currently though?
To be honest, I think you need to log the data from one or the other.
The events are effectively the same, just one happens server side, the
other lets us know the message reached the worker and it also did
something. There are reasons for debugging it is useful to have events
on both sides but for your purposes on webhob, I doubt you need both.
Cheers,
Richard
^ permalink raw reply [flat|nested] 17+ messages in thread
* [PATCH 4/7] bitbake: cooker: add extra recipe information
2013-09-16 13:32 [PATCH v3 0/7] webhob related changes in bitbake, version 3 Alex DAMIAN
` (2 preceding siblings ...)
2013-09-16 13:33 ` [PATCH 3/7] bitbake: runqueue: add runQueueTaskSkipped event Alex DAMIAN
@ 2013-09-16 13:33 ` Alex DAMIAN
2013-09-16 14:10 ` Richard Purdie
2013-09-16 13:33 ` [PATCH 5/7] bitbake: cooker, command: add a command to return global data Alex DAMIAN
` (2 subsequent siblings)
6 siblings, 1 reply; 17+ messages in thread
From: Alex DAMIAN @ 2013-09-16 13:33 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Adding in the extra recipe information when creating
the dependency tree information.
This works in server-mode because the HOB extra_cache
is already enabled. In normal mode, it will do nothing
with no performance impact.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/cooker.py | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 68a3c01..cb0e3e5 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -439,6 +439,29 @@ class BBCooker:
depend_tree["pn"][pn] = {}
depend_tree["pn"][pn]["filename"] = fn
depend_tree["pn"][pn]["version"] = version
+
+ # This data is needed for webhob;
+ # it's here if the bitbake runs in server mode
+ try:
+ summary = self.recipecache.summary[fn]
+ lic = self.recipecache.license[fn]
+ section = self.recipecache.section[fn]
+ description = self.recipecache.description[fn]
+ homepage = self.recipecache.homepage[fn]
+ bugtracker = self.recipecache.bugtracker[fn]
+ inherits = self.recipecache.inherits.get(fn, None)
+ depend_tree["pn"][pn]["filename"] = fn
+ depend_tree["pn"][pn]["version"] = version
+ depend_tree["pn"][pn]["summary"] = summary
+ depend_tree["pn"][pn]["license"] = lic
+ depend_tree["pn"][pn]["section"] = section
+ depend_tree["pn"][pn]["description"] = description
+ depend_tree["pn"][pn]["inherits"] = inherits
+ depend_tree["pn"][pn]["homepage"] = homepage
+ depend_tree["pn"][pn]["bugtracker"] = bugtracker
+ except:
+ pass
+
for dep in rq.rqdata.runq_depends[task]:
depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
deppn = self.recipecache.pkg_fn[depfn]
--
1.8.1.2
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [PATCH 4/7] bitbake: cooker: add extra recipe information
2013-09-16 13:33 ` [PATCH 4/7] bitbake: cooker: add extra recipe information Alex DAMIAN
@ 2013-09-16 14:10 ` Richard Purdie
0 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2013-09-16 14:10 UTC (permalink / raw)
To: Alex DAMIAN; +Cc: bitbake-devel
On Mon, 2013-09-16 at 14:33 +0100, Alex DAMIAN wrote:
> From: Alexandru DAMIAN <alexandru.damian@intel.com>
>
> Adding in the extra recipe information when creating
> the dependency tree information.
>
> This works in server-mode because the HOB extra_cache
> is already enabled. In normal mode, it will do nothing
> with no performance impact.
>
> Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
> ---
> bitbake/lib/bb/cooker.py | 23 +++++++++++++++++++++++
> 1 file changed, 23 insertions(+)
>
> diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
> index 68a3c01..cb0e3e5 100644
> --- a/bitbake/lib/bb/cooker.py
> +++ b/bitbake/lib/bb/cooker.py
> @@ -439,6 +439,29 @@ class BBCooker:
> depend_tree["pn"][pn] = {}
> depend_tree["pn"][pn]["filename"] = fn
> depend_tree["pn"][pn]["version"] = version
> +
> + # This data is needed for webhob;
> + # it's here if the bitbake runs in server mode
> + try:
> + summary = self.recipecache.summary[fn]
> + lic = self.recipecache.license[fn]
> + section = self.recipecache.section[fn]
> + description = self.recipecache.description[fn]
> + homepage = self.recipecache.homepage[fn]
> + bugtracker = self.recipecache.bugtracker[fn]
> + inherits = self.recipecache.inherits.get(fn, None)
> + depend_tree["pn"][pn]["filename"] = fn
> + depend_tree["pn"][pn]["version"] = version
> + depend_tree["pn"][pn]["summary"] = summary
> + depend_tree["pn"][pn]["license"] = lic
> + depend_tree["pn"][pn]["section"] = section
> + depend_tree["pn"][pn]["description"] = description
> + depend_tree["pn"][pn]["inherits"] = inherits
> + depend_tree["pn"][pn]["homepage"] = homepage
> + depend_tree["pn"][pn]["bugtracker"] = bugtracker
> + except:
> + pass
> +
> for dep in rq.rqdata.runq_depends[task]:
> depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
> deppn = self.recipecache.pkg_fn[depfn]
This is horrible, please try and do something like:
diff --git a/bitbake/lib/bb/cache_extra.py b/bitbake/lib/bb/cache_extra.py
index 9e38a43..041ba0e 100644
--- a/bitbake/lib/bb/cache_extra.py
+++ b/bitbake/lib/bb/cache_extra.py
@@ -67,3 +67,6 @@ class HobRecipeInfo(RecipeInfoCommon):
cachedata.bugtracker[fn] = self.bugtracker
cachedata.prevision[fn] = self.prevision
cachedata.files_info[fn] = self.files_info
+
+ @staticmethod
+ def addDepTreePackageInfo(xxx)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 2c8d4dc..84d62e5 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -485,6 +485,10 @@ class BBCooker:
depend_tree["packages"][package]["filename"] = fn
depend_tree["packages"][package]["version"] = version
+ for extracache in self.cooker.caches_array:
+ if <executable function>:
+ <function>(depend_tree["packages"][package])
+
return depend_tree
so the extra cache info is added as a staticmethod function in the extra
cache info file which generated the data. You might as well fix the
hardcoding in the following function marked with:
######## WARNING : this function requires cache_extra to be enabled ########
to use this approach too whilst you're doing this.
Thanks,
Richard
^ permalink raw reply related [flat|nested] 17+ messages in thread
* [PATCH 5/7] bitbake: cooker, command: add a command to return global data
2013-09-16 13:32 [PATCH v3 0/7] webhob related changes in bitbake, version 3 Alex DAMIAN
` (3 preceding siblings ...)
2013-09-16 13:33 ` [PATCH 4/7] bitbake: cooker: add extra recipe information Alex DAMIAN
@ 2013-09-16 13:33 ` Alex DAMIAN
2013-09-16 13:54 ` Richard Purdie
2013-09-16 13:33 ` [PATCH 6/7] bitbake: event: adding events to allow bbclasses to push custom info Alex DAMIAN
2013-09-16 13:33 ` [PATCH 7/7] bitbake: runqueue: add task hash to Queue events Alex DAMIAN
6 siblings, 1 reply; 17+ messages in thread
From: Alex DAMIAN @ 2013-09-16 13:33 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Adding the 'getAllKeysWithFlags' read-only command that will
return a dump of the global data state, together with specified
flags for each key. The flag list is passed in as the first
parameter to the command.
This will be used by UI clients to get the build configuration.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/command.py | 8 ++++++++
bitbake/lib/bb/cooker.py | 15 +++++++++++++++
2 files changed, 23 insertions(+)
diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py
index f1abaf7..5d359e7 100644
--- a/bitbake/lib/bb/command.py
+++ b/bitbake/lib/bb/command.py
@@ -145,6 +145,14 @@ class CommandsSync:
"""
command.cooker.shutdown(True)
+ def getAllKeysWithFlags(self, command, params):
+ """
+ Returns a dump of the global state. Call with
+ variable flags to be retrieved as params.
+ """
+ return command.cooker.getAllKeysWithFlags(params[0])
+ getAllKeysWithFlags.readonly = True
+
def getVariable(self, command, params):
"""
Read the value of a variable from data
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index cb0e3e5..981379b 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -1134,6 +1134,21 @@ class BBCooker:
self.configuration.server_register_idlecallback(buildTargetsIdle, rq)
+
+ def getAllKeysWithFlags(self, flaglist):
+ dump = {}
+ for k in self.data.keys():
+ try:
+ v = self.data.getVar(k, True)
+ if not k.startswith("__") and not bool(self.data.getVarFlag(k, 'func')) and not isinstance(v, bb.data_smart.DataSmart):
+ dump[k] = { 'v' : v }
+ for d in flaglist:
+ dump[k][d] = self.data.getVarFlag(k, d)
+ except:
+ pass
+ return dump
+
+
def generateNewImage(self, image, base_image, package_queue, timestamp, description):
'''
Create a new image with a "require"/"inherit" base_image statement
--
1.8.1.2
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [PATCH 5/7] bitbake: cooker, command: add a command to return global data
2013-09-16 13:33 ` [PATCH 5/7] bitbake: cooker, command: add a command to return global data Alex DAMIAN
@ 2013-09-16 13:54 ` Richard Purdie
0 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2013-09-16 13:54 UTC (permalink / raw)
To: Alex DAMIAN; +Cc: bitbake-devel
On Mon, 2013-09-16 at 14:33 +0100, Alex DAMIAN wrote:
> From: Alexandru DAMIAN <alexandru.damian@intel.com>
>
> Adding the 'getAllKeysWithFlags' read-only command that will
> return a dump of the global data state, together with specified
> flags for each key. The flag list is passed in as the first
> parameter to the command.
>
> This will be used by UI clients to get the build configuration.
>
> Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
> ---
> bitbake/lib/bb/command.py | 8 ++++++++
> bitbake/lib/bb/cooker.py | 15 +++++++++++++++
> 2 files changed, 23 insertions(+)
>
> diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py
> index f1abaf7..5d359e7 100644
> --- a/bitbake/lib/bb/command.py
> +++ b/bitbake/lib/bb/command.py
> @@ -145,6 +145,14 @@ class CommandsSync:
> """
> command.cooker.shutdown(True)
>
> + def getAllKeysWithFlags(self, command, params):
> + """
> + Returns a dump of the global state. Call with
> + variable flags to be retrieved as params.
> + """
> + return command.cooker.getAllKeysWithFlags(params[0])
> + getAllKeysWithFlags.readonly = True
Please match the style of the other commands, i.e.:
+ flaglist = params[0]
+
+ return command.cooker.getAllKeysWithFlags(flaglist)
which then indicates what params[0] actually is. We spell this out in
the functions so people can look at the commands and get some idea of
how to use them without having to look at the code itself.
> def getVariable(self, command, params):
> """
> Read the value of a variable from data
> diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
> index cb0e3e5..981379b 100644
> --- a/bitbake/lib/bb/cooker.py
> +++ b/bitbake/lib/bb/cooker.py
> @@ -1134,6 +1134,21 @@ class BBCooker:
>
> self.configuration.server_register_idlecallback(buildTargetsIdle, rq)
>
> +
> + def getAllKeysWithFlags(self, flaglist):
> + dump = {}
> + for k in self.data.keys():
> + try:
> + v = self.data.getVar(k, True)
> + if not k.startswith("__") and not bool(self.data.getVarFlag(k, 'func')) and not isinstance(v, bb.data_smart.DataSmart):
Please get rid of the "not bool(self.data.getVarFlag(k, 'func')" and
filter your results on the other side of the connection if you need to
do that (you can request the func flag too).
> + dump[k] = { 'v' : v }
> + for d in flaglist:
> + dump[k][d] = self.data.getVarFlag(k, d)
> + except:
which exceptions are we worried about here? Lets be specific please.
> + pass
> + return dump
> +
> +
> def generateNewImage(self, image, base_image, package_queue, timestamp, description):
> '''
> Create a new image with a "require"/"inherit" base_image statement
^ permalink raw reply [flat|nested] 17+ messages in thread
* [PATCH 6/7] bitbake: event: adding events to allow bbclasses to push custom info
2013-09-16 13:32 [PATCH v3 0/7] webhob related changes in bitbake, version 3 Alex DAMIAN
` (4 preceding siblings ...)
2013-09-16 13:33 ` [PATCH 5/7] bitbake: cooker, command: add a command to return global data Alex DAMIAN
@ 2013-09-16 13:33 ` Alex DAMIAN
2013-09-16 13:46 ` Richard Purdie
2013-09-16 13:33 ` [PATCH 7/7] bitbake: runqueue: add task hash to Queue events Alex DAMIAN
6 siblings, 1 reply; 17+ messages in thread
From: Alex DAMIAN @ 2013-09-16 13:33 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
We add a new event that will be triggered by the
package-building class, containing the information
about the currently built package.
Adding an event that will passi information about the
content of each package file that gets built.
Knotty UI will ignore these events by default.
Maybe a mechanism for using a single generic event is in order ?
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/event.py | 16 ++++++++++++++++
bitbake/lib/bb/ui/knotty.py | 2 ++
2 files changed, 18 insertions(+)
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
index 67cfcea..7dd9236 100644
--- a/bitbake/lib/bb/event.py
+++ b/bitbake/lib/bb/event.py
@@ -589,6 +589,22 @@ class PackageInfo(Event):
Event.__init__(self)
self._pkginfolist = pkginfolist
+class SinglePackageInfo(Event):
+ """
+ Single Package Information sent on emit_pkgdata
+ """
+ def __init__(self, data):
+ Event.__init__(self)
+ self._data = data
+
+class PackageFileSizes(Event):
+ """
+ Event that contains information about the file sizes contained in a package.
+ """
+ def __init__(self, data):
+ Event.__init__(self)
+ self._data = data
+
class SanityCheck(Event):
"""
Event to issue sanity check
diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index 0211b50..03baa75 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -477,6 +477,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
# ignore
if isinstance(event, (bb.event.BuildBase,
+ bb.event.PackageFileSizes,
+ bb.event.SinglePackageInfo,
bb.event.StampUpdate,
bb.event.ConfigParsed,
bb.event.RecipeParsed,
--
1.8.1.2
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [PATCH 6/7] bitbake: event: adding events to allow bbclasses to push custom info
2013-09-16 13:33 ` [PATCH 6/7] bitbake: event: adding events to allow bbclasses to push custom info Alex DAMIAN
@ 2013-09-16 13:46 ` Richard Purdie
2013-09-16 13:56 ` Damian, Alexandru
0 siblings, 1 reply; 17+ messages in thread
From: Richard Purdie @ 2013-09-16 13:46 UTC (permalink / raw)
To: Alex DAMIAN; +Cc: bitbake-devel
On Mon, 2013-09-16 at 14:33 +0100, Alex DAMIAN wrote:
> From: Alexandru DAMIAN <alexandru.damian@intel.com>
>
> We add a new event that will be triggered by the
> package-building class, containing the information
> about the currently built package.
>
> Adding an event that will passi information about the
> content of each package file that gets built.
>
> Knotty UI will ignore these events by default.
>
> Maybe a mechanism for using a single generic event is in order ?
>
> Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
> ---
> bitbake/lib/bb/event.py | 16 ++++++++++++++++
> bitbake/lib/bb/ui/knotty.py | 2 ++
> 2 files changed, 18 insertions(+)
>
> diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
> index 67cfcea..7dd9236 100644
> --- a/bitbake/lib/bb/event.py
> +++ b/bitbake/lib/bb/event.py
> @@ -589,6 +589,22 @@ class PackageInfo(Event):
> Event.__init__(self)
> self._pkginfolist = pkginfolist
>
> +class SinglePackageInfo(Event):
> + """
> + Single Package Information sent on emit_pkgdata
> + """
> + def __init__(self, data):
> + Event.__init__(self)
> + self._data = data
> +
> +class PackageFileSizes(Event):
> + """
> + Event that contains information about the file sizes contained in a package.
> + """
> + def __init__(self, data):
> + Event.__init__(self)
> + self._data = data
> +
> class SanityCheck(Event):
> """
Your explanation in the commit message isn't as detailed as it could
be.
I'm guessing but I suspect the problem we have is that the event cannot
be rebuilt on the other side of an XMLRPC connection without having the
event class in a common namespace?
This is therefore a hack to allow the event to exist in all the client
side namespaces.
Ideally we should really have an API which allows the classes themselves
to declare their event formats and not require adding to event.py every
time a new piece of code/event is added.
Presumably there is some change required to OE-Core in order for this
get used to?
Is the PackageFileSizes not contained in the other event data?
I appreciate we allowed PackageInfo in but I didn't like it at the time,
I like it even less now I see we're just going to keep doing this :(.
> Event to issue sanity check
> diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
> index 0211b50..03baa75 100644
> --- a/bitbake/lib/bb/ui/knotty.py
> +++ b/bitbake/lib/bb/ui/knotty.py
> @@ -477,6 +477,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
>
> # ignore
> if isinstance(event, (bb.event.BuildBase,
> + bb.event.PackageFileSizes,
> + bb.event.SinglePackageInfo,
> bb.event.StampUpdate,
> bb.event.ConfigParsed,
> bb.event.RecipeParsed,
Given the event masking that is now in knotty, do we need to do this?
Can't we remove this block?
Cheers,
Richard
^ permalink raw reply [flat|nested] 17+ messages in thread
* Re: [PATCH 6/7] bitbake: event: adding events to allow bbclasses to push custom info
2013-09-16 13:46 ` Richard Purdie
@ 2013-09-16 13:56 ` Damian, Alexandru
2013-09-16 15:20 ` Richard Purdie
0 siblings, 1 reply; 17+ messages in thread
From: Damian, Alexandru @ 2013-09-16 13:56 UTC (permalink / raw)
To: Richard Purdie; +Cc: bitbake-devel
[-- Attachment #1: Type: text/plain, Size: 4512 bytes --]
On Mon, Sep 16, 2013 at 2:46 PM, Richard Purdie <
richard.purdie@linuxfoundation.org> wrote:
> On Mon, 2013-09-16 at 14:33 +0100, Alex DAMIAN wrote:
> > From: Alexandru DAMIAN <alexandru.damian@intel.com>
> >
> > We add a new event that will be triggered by the
> > package-building class, containing the information
> > about the currently built package.
> >
> > Adding an event that will passi information about the
> > content of each package file that gets built.
> >
> > Knotty UI will ignore these events by default.
> >
> > Maybe a mechanism for using a single generic event is in order ?
> >
> > Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
> > ---
> > bitbake/lib/bb/event.py | 16 ++++++++++++++++
> > bitbake/lib/bb/ui/knotty.py | 2 ++
> > 2 files changed, 18 insertions(+)
> >
> > diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
> > index 67cfcea..7dd9236 100644
> > --- a/bitbake/lib/bb/event.py
> > +++ b/bitbake/lib/bb/event.py
> > @@ -589,6 +589,22 @@ class PackageInfo(Event):
> > Event.__init__(self)
> > self._pkginfolist = pkginfolist
> >
> > +class SinglePackageInfo(Event):
> > + """
> > + Single Package Information sent on emit_pkgdata
> > + """
> > + def __init__(self, data):
> > + Event.__init__(self)
> > + self._data = data
> > +
> > +class PackageFileSizes(Event):
> > + """
> > + Event that contains information about the file sizes contained in a
> package.
> > + """
> > + def __init__(self, data):
> > + Event.__init__(self)
> > + self._data = data
> > +
> > class SanityCheck(Event):
> > """
>
> Your explanation in the commit message isn't as detailed as it could
> be.
>
> I'm guessing but I suspect the problem we have is that the event cannot
> be rebuilt on the other side of an XMLRPC connection without having the
> event class in a common namespace?
>
> This is therefore a hack to allow the event to exist in all the client
> side namespaces.
>
[Alex] Yep, this is an ugly hack.
>
> Ideally we should really have an API which allows the classes themselves
> to declare their event formats and not require adding to event.py every
> time a new piece of code/event is added.
>
[Alex] I would favor a single "GenericEvent" that will take a set of
defined parameters,
that would allow "subclassing" at runtime - sort of introspection if you
will.
Something like this:
class GenericInfo(Event):
def __init__(self, type, data):
Event.__init__(self)
self._type = type
self._data = data
with a call like
bb.event.fire(bb.event.GenericInfo("PackageInfo", pkginfolist), e.data)
the UI handlers / anybody would subscribe to GenericInfo events,
and the processing code would look like:
if isinstance(event, bb.event.GenericInfo):
if event._type == "PackageInfo":
savePackageInfo(event)
elif event._type == "PackageFileSizes":
....
What do you think ?
>
> Presumably there is some change required to OE-Core in order for this
> get used to?
>
The code using the e
>
> Is the PackageFileSizes not contained in the other event data?
>
[Alex] Not that I could find.
>
> I appreciate we allowed PackageInfo in but I didn't like it at the time,
> I like it even less now I see we're just going to keep doing this :(.
>
> > Event to issue sanity check
> > diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
> > index 0211b50..03baa75 100644
> > --- a/bitbake/lib/bb/ui/knotty.py
> > +++ b/bitbake/lib/bb/ui/knotty.py
> > @@ -477,6 +477,8 @@ def main(server, eventHandler, params, tf =
> TerminalFilter):
> >
> > # ignore
> > if isinstance(event, (bb.event.BuildBase,
> > + bb.event.PackageFileSizes,
> > + bb.event.SinglePackageInfo,
> > bb.event.StampUpdate,
> > bb.event.ConfigParsed,
> > bb.event.RecipeParsed,
>
>
> Given the event masking that is now in knotty, do we need to do this?
> Can't we remove this block?
>
[Alex] Yep, it should get removed. I refrained from doing so since it
wasn't deleted
when the subscription code got merged in.
> Cheers,
>
> Richard
>
>
>
--
Alex Damian
Yocto Project
SSG / OTC
[-- Attachment #2: Type: text/html, Size: 7749 bytes --]
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 6/7] bitbake: event: adding events to allow bbclasses to push custom info
2013-09-16 13:56 ` Damian, Alexandru
@ 2013-09-16 15:20 ` Richard Purdie
0 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2013-09-16 15:20 UTC (permalink / raw)
To: Damian, Alexandru; +Cc: bitbake-devel
On Mon, 2013-09-16 at 14:56 +0100, Damian, Alexandru wrote:
> On Mon, Sep 16, 2013 at 2:46 PM, Richard Purdie
> <richard.purdie@linuxfoundation.org> wrote:
> Your explanation in the commit message isn't as detailed as it could
> be.
>
> I'm guessing but I suspect the problem we have is that the
> event cannot
> be rebuilt on the other side of an XMLRPC connection without
> having the
> event class in a common namespace?
>
> This is therefore a hack to allow the event to exist in all
> the client
> side namespaces.
>
>
> [Alex] Yep, this is an ugly hack.
In future *please* put this kind of information into commit messages.
I'm having a lot of trouble with these patches as the commit messages
don't give me all the information I need to review them.
>
>
> Ideally we should really have an API which allows the classes
> themselves
> to declare their event formats and not require adding to
> event.py every
> time a new piece of code/event is added.
>
> [Alex] I would favor a single "GenericEvent" that will take a set of
> defined parameters,
> that would allow "subclassing" at runtime - sort of introspection if
> you will.
>
> Something like this:
>
>
> class GenericInfo(Event):
>
> def __init__(self, type, data):
> Event.__init__(self)
> self._type = type
> self._data = data
>
> with a call like
>
>
> bb.event.fire(bb.event.GenericInfo("PackageInfo", pkginfolist),
> e.data)
>
>
> the UI handlers / anybody would subscribe to GenericInfo events,
> and the processing code would look like:
>
>
> if isinstance(event, bb.event.GenericInfo):
> if event._type == "PackageInfo":
> savePackageInfo(event)
> elif event._type == "PackageFileSizes":
> ....
>
Lets do it but can you:
a) Stop using the underscores please. I know some older events have them
but the data is not internal so lets not persist broken syntax going
forward.
b) Call this MetadataEvent and put in a header comment explaining what
it is and why/when you'd use it.
c) Convert the user of PackageInfo in OE-Core to use the new event if it
is available, falling back to PackageInfo.
d) Update hob to accept the new event
c/d can come later but I do want to see patches for them ultimately.
We'll remove PackageInfo in the 1.6 cycle.
Cheers,
Richard
^ permalink raw reply [flat|nested] 17+ messages in thread
* [PATCH 7/7] bitbake: runqueue: add task hash to Queue events
2013-09-16 13:32 [PATCH v3 0/7] webhob related changes in bitbake, version 3 Alex DAMIAN
` (5 preceding siblings ...)
2013-09-16 13:33 ` [PATCH 6/7] bitbake: event: adding events to allow bbclasses to push custom info Alex DAMIAN
@ 2013-09-16 13:33 ` Alex DAMIAN
6 siblings, 0 replies; 17+ messages in thread
From: Alex DAMIAN @ 2013-09-16 13:33 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Adding the sstate-related hash for all runqueue and
scenequeue tasks, as it's needed in the WebHob data.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/runqueue.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 2ab4405..18ebce0 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -223,6 +223,9 @@ class RunQueueData:
def get_task_file(self, task):
return self.taskData.fn_index[self.runq_fnid[task]]
+ def get_task_hash(self, task):
+ return self.runq_hash[task]
+
def get_user_idstring(self, task, task_name_suffix = ""):
fn = self.taskData.fn_index[self.runq_fnid[task]]
taskname = self.runq_task[task] + task_name_suffix
@@ -1792,6 +1795,7 @@ class runQueueEvent(bb.event.Event):
self.taskstring = rq.rqdata.get_user_idstring(task)
self.taskname = rq.rqdata.get_task_name(task)
self.taskfile = rq.rqdata.get_task_file(task)
+ self.taskhash = rq.rqdata.get_task_hash(task)
self.stats = stats.copy()
bb.event.Event.__init__(self)
@@ -1805,6 +1809,7 @@ class sceneQueueEvent(runQueueEvent):
self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene"
self.taskfile = rq.rqdata.get_task_file(realtask)
+ self.taskhash = rq.rqdata.get_task_hash(task)
class runQueueTaskStarted(runQueueEvent):
"""
--
1.8.1.2
^ permalink raw reply related [flat|nested] 17+ messages in thread