* [PATCH 01/13] bitbake: cooker: clean up code and avoid duplication
2013-09-09 16:40 [PATCH 00/13] webhob related changes in bitbake Alex DAMIAN
@ 2013-09-09 16:40 ` Alex DAMIAN
2013-09-09 16:40 ` [PATCH 02/13] bitbake: runqueue: Add runQueueTaskSkipped event Alex DAMIAN
` (11 subsequent siblings)
12 siblings, 0 replies; 18+ messages in thread
From: Alex DAMIAN @ 2013-09-09 16:40 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Clean-up to avoid duplication and promote code reuse.
We're using the prepareTreeData in all instances where
this needs to be done.
When we're only testing target image contents (HOB),
we do not abort with an error if the target is
unbuildable.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/cooker.py | 30 +++++++++---------------------
1 file changed, 9 insertions(+), 21 deletions(-)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index f03f16b..9fd3417 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -401,9 +401,7 @@ class BBCooker:
localdata = data.createCopy(self.data)
bb.data.update_data(localdata)
bb.data.expandKeys(localdata)
- # We set abort to False here to prevent unbuildable targets raising
- # an exception when we're just generating data
- taskdata = bb.taskdata.TaskData(False, skiplist=self.skiplist)
+ taskdata = bb.taskdata.TaskData(self.configuration.abort, skiplist=self.skiplist)
runlist = []
current = 0
@@ -495,7 +493,12 @@ class BBCooker:
"""
Create a dependency tree of pkgs_to_build, returning the data.
"""
+
+ # We set abort to False here to prevent unbuildable targets raising
+ # an exception when we're just generating data
+ self.configuration.abort = False
_, taskdata = self.prepareTreeData(pkgs_to_build, task)
+
tasks_fnid = []
if len(taskdata.tasks_name) != 0:
for task in xrange(len(taskdata.tasks_name)):
@@ -1076,13 +1079,6 @@ class BBCooker:
Attempt to build the targets specified
"""
- # If we are told to do the NULL task then query the default task
- if (task == None):
- task = self.configuration.cmd
-
- universe = ('universe' in targets)
- targets = self.checkPackages(targets)
-
def buildTargetsIdle(server, rq, abort):
if abort or self.state == state.stop:
rq.finish_runqueue(True)
@@ -1109,21 +1105,13 @@ class BBCooker:
self.buildSetVars()
buildname = self.data.getVar("BUILDNAME")
- bb.event.fire(bb.event.BuildStarted(buildname, targets), self.data)
-
- localdata = data.createCopy(self.data)
- bb.data.update_data(localdata)
- bb.data.expandKeys(localdata)
- taskdata = bb.taskdata.TaskData(self.configuration.abort, skiplist=self.skiplist)
+ bb.event.fire(bb.event.BuildStarted(buildname, targets), self.data)
- runlist = []
- for k in targets:
- taskdata.add_provider(localdata, self.recipecache, k)
- runlist.append([k, "do_%s" % task])
- taskdata.add_unresolved(localdata, self.recipecache)
+ runlist, taskdata = self.prepareTreeData(targets, task)
rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist)
+ universe = ('universe' in targets)
if universe:
rq.rqdata.warn_multi_bb = True
--
1.8.1.2
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 02/13] bitbake: runqueue: Add runQueueTaskSkipped event
2013-09-09 16:40 [PATCH 00/13] webhob related changes in bitbake Alex DAMIAN
2013-09-09 16:40 ` [PATCH 01/13] bitbake: cooker: clean up code and avoid duplication Alex DAMIAN
@ 2013-09-09 16:40 ` Alex DAMIAN
2013-09-09 16:40 ` [PATCH 03/13] bitbake: runqueue: added prebuild detailed information Alex DAMIAN
` (10 subsequent siblings)
12 siblings, 0 replies; 18+ messages in thread
From: Alex DAMIAN @ 2013-09-09 16:40 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Adding a runQueueTaskSkipped to notify all listeners
about the tasks that are not run either because they
are set-scened or they don't need an update (timestamps
are ok)
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/runqueue.py | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index a868332..f084407 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -217,6 +217,12 @@ class RunQueueData:
ret.extend([nam])
return ret
+ def get_task_name(self, task):
+ return self.runq_task[task]
+
+ def get_task_package(self, task):
+ return self.taskData.fn_index[self.runq_fnid[task]]
+
def get_user_idstring(self, task, task_name_suffix = ""):
fn = self.taskData.fn_index[self.runq_fnid[task]]
taskname = self.runq_task[task] + task_name_suffix
@@ -1328,6 +1334,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
def task_skip(self, task):
self.runq_running[task] = 1
self.runq_buildable[task] = 1
+ bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq), self.cfgData)
self.task_completeoutright(task)
self.stats.taskCompleted()
self.stats.taskSkipped()
@@ -1828,6 +1835,15 @@ class runQueueTaskCompleted(runQueueEvent):
Event notifing a task completed
"""
+class runQueueTaskSkipped(runQueueEvent):
+ """
+ Event notifing a task was skipped
+ """
+ def __init__(self, task, stats, rq):
+ runQueueEvent.__init__(self, task, stats, rq)
+ self._package = rq.rqdata.get_task_package(task)
+ self._task = rq.rqdata.get_task_name(task)
+
class runQueuePipe():
"""
Abstraction for a pipe between a worker thread and the server
--
1.8.1.2
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 03/13] bitbake: runqueue: added prebuild detailed information
2013-09-09 16:40 [PATCH 00/13] webhob related changes in bitbake Alex DAMIAN
2013-09-09 16:40 ` [PATCH 01/13] bitbake: cooker: clean up code and avoid duplication Alex DAMIAN
2013-09-09 16:40 ` [PATCH 02/13] bitbake: runqueue: Add runQueueTaskSkipped event Alex DAMIAN
@ 2013-09-09 16:40 ` Alex DAMIAN
2013-09-09 16:40 ` [PATCH 04/13] bitbake: runqueue, build: event data change Alex DAMIAN
` (9 subsequent siblings)
12 siblings, 0 replies; 18+ messages in thread
From: Alex DAMIAN @ 2013-09-09 16:40 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Adds why the prebuild tasks were skipped,
as to distinguish about covered/existing tasks.
Adds task name and task package to all runQueue
and sceneQueue events.
---
bitbake/lib/bb/runqueue.py | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index f084407..002bf76 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -1331,10 +1331,10 @@ class RunQueueExecuteTasks(RunQueueExecute):
if self.rqdata.taskData.abort:
self.rq.state = runQueueCleanUp
- def task_skip(self, task):
+ def task_skip(self, task, skip_reason = ""):
self.runq_running[task] = 1
self.runq_buildable[task] = 1
- bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq), self.cfgData)
+ bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, skip_reason), self.cfgData)
self.task_completeoutright(task)
self.stats.taskCompleted()
self.stats.taskSkipped()
@@ -1359,13 +1359,13 @@ class RunQueueExecuteTasks(RunQueueExecute):
if task in self.rq.scenequeue_covered:
logger.debug(2, "Setscene covered task %s (%s)", task,
self.rqdata.get_user_idstring(task))
- self.task_skip(task)
+ self.task_skip(task, "covered")
return True
if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
logger.debug(2, "Stamp current task %s (%s)", task,
self.rqdata.get_user_idstring(task))
- self.task_skip(task)
+ self.task_skip(task, "existing")
return True
taskdep = self.rqdata.dataCache.task_deps[fn]
@@ -1786,6 +1786,8 @@ class runQueueEvent(bb.event.Event):
def __init__(self, task, stats, rq):
self.taskid = task
self.taskstring = rq.rqdata.get_user_idstring(task)
+ self.taskname = rq.rqdata.get_task_name(task)
+ self.taskpackage = rq.rqdata.get_task_package(task)
self.stats = stats.copy()
bb.event.Event.__init__(self)
@@ -1797,6 +1799,8 @@ class sceneQueueEvent(runQueueEvent):
runQueueEvent.__init__(self, task, stats, rq)
realtask = rq.rqdata.runq_setscene[task]
self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
+ self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene"
+ self.taskpackage = rq.rqdata.get_task_package(realtask)
class runQueueTaskStarted(runQueueEvent):
"""
@@ -1839,10 +1843,11 @@ class runQueueTaskSkipped(runQueueEvent):
"""
Event notifing a task was skipped
"""
- def __init__(self, task, stats, rq):
+ def __init__(self, task, stats, rq, skip_reason):
runQueueEvent.__init__(self, task, stats, rq)
self._package = rq.rqdata.get_task_package(task)
self._task = rq.rqdata.get_task_name(task)
+ self._skip = skip_reason
class runQueuePipe():
"""
--
1.8.1.2
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 04/13] bitbake: runqueue, build: event data change
2013-09-09 16:40 [PATCH 00/13] webhob related changes in bitbake Alex DAMIAN
` (2 preceding siblings ...)
2013-09-09 16:40 ` [PATCH 03/13] bitbake: runqueue: added prebuild detailed information Alex DAMIAN
@ 2013-09-09 16:40 ` Alex DAMIAN
2013-09-10 13:52 ` Richard Purdie
2013-09-09 16:40 ` [PATCH 05/13] bitbake: runqueue: add sceneQueueTaskCompleted event Alex DAMIAN
` (8 subsequent siblings)
12 siblings, 1 reply; 18+ messages in thread
From: Alex DAMIAN @ 2013-09-09 16:40 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
We modify event data to make sure we can match build events
with runqueue events. This is important because the build
events supply datastore-evaluated data while runqueue events supply
bitbake-related data. We get the recipe name and version from
the datastore, while the file paths and timing information are
collected from the RunQueue.
We modify filling-in the information side to take advantage of the
new info supplied.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/build.py | 3 +++
bitbake/lib/bb/runqueue.py | 8 ++++----
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index a53aba9..1265bdc 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -72,6 +72,9 @@ class TaskBase(event.Event):
def __init__(self, t, d ):
self._task = t
self._package = d.getVar("PF", True)
+ self._file = d.getVar("FILE", True)
+ self.taskfile = self._file
+ self.taskname = self._task
event.Event.__init__(self)
self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 002bf76..73dd059 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -220,7 +220,7 @@ class RunQueueData:
def get_task_name(self, task):
return self.runq_task[task]
- def get_task_package(self, task):
+ def get_task_file(self, task):
return self.taskData.fn_index[self.runq_fnid[task]]
def get_user_idstring(self, task, task_name_suffix = ""):
@@ -1787,7 +1787,7 @@ class runQueueEvent(bb.event.Event):
self.taskid = task
self.taskstring = rq.rqdata.get_user_idstring(task)
self.taskname = rq.rqdata.get_task_name(task)
- self.taskpackage = rq.rqdata.get_task_package(task)
+ self.taskfile = rq.rqdata.get_task_file(task)
self.stats = stats.copy()
bb.event.Event.__init__(self)
@@ -1800,7 +1800,7 @@ class sceneQueueEvent(runQueueEvent):
realtask = rq.rqdata.runq_setscene[task]
self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene"
- self.taskpackage = rq.rqdata.get_task_package(realtask)
+ self.taskfile = rq.rqdata.get_task_file(realtask)
class runQueueTaskStarted(runQueueEvent):
"""
@@ -1845,7 +1845,7 @@ class runQueueTaskSkipped(runQueueEvent):
"""
def __init__(self, task, stats, rq, skip_reason):
runQueueEvent.__init__(self, task, stats, rq)
- self._package = rq.rqdata.get_task_package(task)
+ self._file = rq.rqdata.get_task_file(task)
self._task = rq.rqdata.get_task_name(task)
self._skip = skip_reason
--
1.8.1.2
^ permalink raw reply related [flat|nested] 18+ messages in thread* Re: [PATCH 04/13] bitbake: runqueue, build: event data change
2013-09-09 16:40 ` [PATCH 04/13] bitbake: runqueue, build: event data change Alex DAMIAN
@ 2013-09-10 13:52 ` Richard Purdie
0 siblings, 0 replies; 18+ messages in thread
From: Richard Purdie @ 2013-09-10 13:52 UTC (permalink / raw)
To: Alex DAMIAN; +Cc: bitbake-devel
On Mon, 2013-09-09 at 17:40 +0100, Alex DAMIAN wrote:
> From: Alexandru DAMIAN <alexandru.damian@intel.com>
>
> We modify event data to make sure we can match build events
> with runqueue events. This is important because the build
> events supply datastore-evaluated data while runqueue events supply
> bitbake-related data. We get the recipe name and version from
> the datastore, while the file paths and timing information are
> collected from the RunQueue.
>
> We modify filling-in the information side to take advantage of the
> new info supplied.
>
> Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
> ---
> bitbake/lib/bb/build.py | 3 +++
> bitbake/lib/bb/runqueue.py | 8 ++++----
> 2 files changed, 7 insertions(+), 4 deletions(-)
>
> diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
> index a53aba9..1265bdc 100644
> --- a/bitbake/lib/bb/build.py
> +++ b/bitbake/lib/bb/build.py
> @@ -72,6 +72,9 @@ class TaskBase(event.Event):
> def __init__(self, t, d ):
> self._task = t
> self._package = d.getVar("PF", True)
> + self._file = d.getVar("FILE", True)
> + self.taskfile = self._file
> + self.taskname = self._task
> event.Event.__init__(self)
> self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
>
> diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
> index 002bf76..73dd059 100644
> --- a/bitbake/lib/bb/runqueue.py
> +++ b/bitbake/lib/bb/runqueue.py
> @@ -220,7 +220,7 @@ class RunQueueData:
> def get_task_name(self, task):
> return self.runq_task[task]
>
> - def get_task_package(self, task):
> + def get_task_file(self, task):
> return self.taskData.fn_index[self.runq_fnid[task]]
>
> def get_user_idstring(self, task, task_name_suffix = ""):
> @@ -1787,7 +1787,7 @@ class runQueueEvent(bb.event.Event):
> self.taskid = task
> self.taskstring = rq.rqdata.get_user_idstring(task)
> self.taskname = rq.rqdata.get_task_name(task)
> - self.taskpackage = rq.rqdata.get_task_package(task)
> + self.taskfile = rq.rqdata.get_task_file(task)
> self.stats = stats.copy()
> bb.event.Event.__init__(self)
>
> @@ -1800,7 +1800,7 @@ class sceneQueueEvent(runQueueEvent):
> realtask = rq.rqdata.runq_setscene[task]
> self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
> self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene"
> - self.taskpackage = rq.rqdata.get_task_package(realtask)
> + self.taskfile = rq.rqdata.get_task_file(realtask)
>
> class runQueueTaskStarted(runQueueEvent):
> """
> @@ -1845,7 +1845,7 @@ class runQueueTaskSkipped(runQueueEvent):
> """
> def __init__(self, task, stats, rq, skip_reason):
> runQueueEvent.__init__(self, task, stats, rq)
> - self._package = rq.rqdata.get_task_package(task)
> + self._file = rq.rqdata.get_task_file(task)
> self._task = rq.rqdata.get_task_name(task)
> self._skip = skip_reason
>
You added get_task_package() in 2/13, added calls to in 3/13, now we're
getting rid of it. Is there not some way to clean up these patches so we
just add the right things? Is get_task_package() still needed? Reviewing
patches where things get added then removed again is rather confusing...
Cheers,
Richard
^ permalink raw reply [flat|nested] 18+ messages in thread
* [PATCH 05/13] bitbake: runqueue: add sceneQueueTaskCompleted event
2013-09-09 16:40 [PATCH 00/13] webhob related changes in bitbake Alex DAMIAN
` (3 preceding siblings ...)
2013-09-09 16:40 ` [PATCH 04/13] bitbake: runqueue, build: event data change Alex DAMIAN
@ 2013-09-09 16:40 ` Alex DAMIAN
2013-09-09 16:40 ` [PATCH 06/13] bitbake: build: add link to logfile for Task events Alex DAMIAN
` (7 subsequent siblings)
12 siblings, 0 replies; 18+ messages in thread
From: Alex DAMIAN @ 2013-09-09 16:40 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Adding an event to be fired when a scene task is completed.
It is analogous to the run task completed event, and has
been missing for some reason.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/runqueue.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 73dd059..11ff193 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -1652,6 +1652,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
def task_complete(self, task):
self.stats.taskCompleted()
+ bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
self.task_completeoutright(task)
def task_fail(self, task, result):
@@ -1839,6 +1840,11 @@ class runQueueTaskCompleted(runQueueEvent):
Event notifing a task completed
"""
+class sceneQueueTaskCompleted(sceneQueueEvent):
+ """
+ Event notifing a setscene task completed
+ """
+
class runQueueTaskSkipped(runQueueEvent):
"""
Event notifing a task was skipped
--
1.8.1.2
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 06/13] bitbake: build: add link to logfile for Task events
2013-09-09 16:40 [PATCH 00/13] webhob related changes in bitbake Alex DAMIAN
` (4 preceding siblings ...)
2013-09-09 16:40 ` [PATCH 05/13] bitbake: runqueue: add sceneQueueTaskCompleted event Alex DAMIAN
@ 2013-09-09 16:40 ` Alex DAMIAN
2013-09-10 14:01 ` Richard Purdie
2013-09-09 16:40 ` [PATCH 07/13] bitbake: cooker, command: add a command to return global data Alex DAMIAN
` (6 subsequent siblings)
12 siblings, 1 reply; 18+ messages in thread
From: Alex DAMIAN @ 2013-09-09 16:40 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
We add a pointer to the logfile for Task events, so
that we can trace that in the Bitbake database
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/build.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index 1265bdc..4abbe48 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -73,6 +73,7 @@ class TaskBase(event.Event):
self._task = t
self._package = d.getVar("PF", True)
self._file = d.getVar("FILE", True)
+ self.logfile = d.getVar("BB_LOGFILE", True)
self.taskfile = self._file
self.taskname = self._task
event.Event.__init__(self)
--
1.8.1.2
^ permalink raw reply related [flat|nested] 18+ messages in thread* Re: [PATCH 06/13] bitbake: build: add link to logfile for Task events
2013-09-09 16:40 ` [PATCH 06/13] bitbake: build: add link to logfile for Task events Alex DAMIAN
@ 2013-09-10 14:01 ` Richard Purdie
0 siblings, 0 replies; 18+ messages in thread
From: Richard Purdie @ 2013-09-10 14:01 UTC (permalink / raw)
To: Alex DAMIAN; +Cc: bitbake-devel
On Mon, 2013-09-09 at 17:40 +0100, Alex DAMIAN wrote:
> From: Alexandru DAMIAN <alexandru.damian@intel.com>
>
> We add a pointer to the logfile for Task events, so
> that we can trace that in the Bitbake database
>
> Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
> ---
> bitbake/lib/bb/build.py | 1 +
> 1 file changed, 1 insertion(+)
>
> diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
> index 1265bdc..4abbe48 100644
> --- a/bitbake/lib/bb/build.py
> +++ b/bitbake/lib/bb/build.py
> @@ -73,6 +73,7 @@ class TaskBase(event.Event):
> self._task = t
> self._package = d.getVar("PF", True)
> self._file = d.getVar("FILE", True)
> + self.logfile = d.getVar("BB_LOGFILE", True)
> self.taskfile = self._file
> self.taskname = self._task
> event.Event.__init__(self)
This leaves the TaskFailed() events still manually passing in logfile as
a separate parameter. I think we might as well adjust the code to always
pass the logfile to the events instead of this, mainly as it makes it
explictly clear where and when the logfile is uses. BB_LOGFILE is really
meant more for external usage than for bitbake itself.
Cheers,
Richard
^ permalink raw reply [flat|nested] 18+ messages in thread
* [PATCH 07/13] bitbake: cooker, command: add a command to return global data
2013-09-09 16:40 [PATCH 00/13] webhob related changes in bitbake Alex DAMIAN
` (5 preceding siblings ...)
2013-09-09 16:40 ` [PATCH 06/13] bitbake: build: add link to logfile for Task events Alex DAMIAN
@ 2013-09-09 16:40 ` Alex DAMIAN
2013-09-13 14:52 ` Richard Purdie
2013-09-09 16:40 ` [PATCH 08/13] bitbake: cooker: use prepareTreeData when computing environment Alex DAMIAN
` (5 subsequent siblings)
12 siblings, 1 reply; 18+ messages in thread
From: Alex DAMIAN @ 2013-09-09 16:40 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Adding the 'getDataDump' read-only command that will
return a dump of the global data state.
This will be used by the DSI to store the build configuration.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/command.py | 7 +++++++
bitbake/lib/bb/cooker.py | 13 +++++++++++++
2 files changed, 20 insertions(+)
diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py
index 3527ad0..4d36c4b 100644
--- a/bitbake/lib/bb/command.py
+++ b/bitbake/lib/bb/command.py
@@ -145,6 +145,13 @@ class CommandsSync:
"""
command.cooker.stop()
+ def getDataDump(self, command, params):
+ """
+ Returns a dump of the global state.
+ """
+ return command.cooker.getGlobalExpandedData()
+ getDataDump.readonly = True
+
def getVariable(self, command, params):
"""
Read the value of a variable from data
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 9fd3417..383c9c4 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -1117,6 +1117,19 @@ class BBCooker:
self.configuration.server_register_idlecallback(buildTargetsIdle, rq)
+
+ def getGlobalExpandedData(self):
+ dump = {}
+ for k in self.data.keys():
+ try:
+ v = self.data.getVar(k, True)
+ if not k.startswith("__") and not bool(self.data.getVarFlag(k, 'func')) and not isinstance(v, bb.data_smart.DataSmart):
+ dump[k] = { 'v' : v, 'd' : self.data.getVarFlag(k, 'doc') }
+ except:
+ pass
+ return dump
+
+
def generateNewImage(self, image, base_image, package_queue, timestamp, description):
'''
Create a new image with a "require"/"inherit" base_image statement
--
1.8.1.2
^ permalink raw reply related [flat|nested] 18+ messages in thread* Re: [PATCH 07/13] bitbake: cooker, command: add a command to return global data
2013-09-09 16:40 ` [PATCH 07/13] bitbake: cooker, command: add a command to return global data Alex DAMIAN
@ 2013-09-13 14:52 ` Richard Purdie
0 siblings, 0 replies; 18+ messages in thread
From: Richard Purdie @ 2013-09-13 14:52 UTC (permalink / raw)
To: Alex DAMIAN; +Cc: bitbake-devel
On Mon, 2013-09-09 at 17:40 +0100, Alex DAMIAN wrote:
> From: Alexandru DAMIAN <alexandru.damian@intel.com>
>
> Adding the 'getDataDump' read-only command that will
> return a dump of the global data state.
>
> This will be used by the DSI to store the build configuration.
>
> Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
> ---
> bitbake/lib/bb/command.py | 7 +++++++
> bitbake/lib/bb/cooker.py | 13 +++++++++++++
> 2 files changed, 20 insertions(+)
>
> diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py
> index 3527ad0..4d36c4b 100644
> --- a/bitbake/lib/bb/command.py
> +++ b/bitbake/lib/bb/command.py
> @@ -145,6 +145,13 @@ class CommandsSync:
> """
> command.cooker.stop()
>
> + def getDataDump(self, command, params):
> + """
> + Returns a dump of the global state.
> + """
> + return command.cooker.getGlobalExpandedData()
> + getDataDump.readonly = True
> +
> def getVariable(self, command, params):
> """
> Read the value of a variable from data
> diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
> index 9fd3417..383c9c4 100644
> --- a/bitbake/lib/bb/cooker.py
> +++ b/bitbake/lib/bb/cooker.py
> @@ -1117,6 +1117,19 @@ class BBCooker:
>
> self.configuration.server_register_idlecallback(buildTargetsIdle, rq)
>
> +
> + def getGlobalExpandedData(self):
> + dump = {}
> + for k in self.data.keys():
> + try:
> + v = self.data.getVar(k, True)
> + if not k.startswith("__") and not bool(self.data.getVarFlag(k, 'func')) and not isinstance(v, bb.data_smart.DataSmart):
> + dump[k] = { 'v' : v, 'd' : self.data.getVarFlag(k, 'doc') }
> + except:
> + pass
> + return dump
There is a 'doc' hardcoded into there, perhaps this should be a better
named command and take a parameter?
Cheers,
Richard
^ permalink raw reply [flat|nested] 18+ messages in thread
* [PATCH 08/13] bitbake: cooker: use prepareTreeData when computing environment
2013-09-09 16:40 [PATCH 00/13] webhob related changes in bitbake Alex DAMIAN
` (6 preceding siblings ...)
2013-09-09 16:40 ` [PATCH 07/13] bitbake: cooker, command: add a command to return global data Alex DAMIAN
@ 2013-09-09 16:40 ` Alex DAMIAN
2013-09-09 16:41 ` [PATCH 09/13] bitbake: event: do not abort on UI handlers error Alex DAMIAN
` (4 subsequent siblings)
12 siblings, 0 replies; 18+ messages in thread
From: Alex DAMIAN @ 2013-09-09 16:40 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Eliminate code duplication when building the task data
and run list environment, using the prepareTreeData function
that is used elsewhere.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/cooker.py | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 383c9c4..1eb1768 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -348,13 +348,7 @@ class BBCooker:
if pkgs_to_build[0] in set(ignore.split()):
bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
- localdata = data.createCopy(self.data)
- bb.data.update_data(localdata)
- bb.data.expandKeys(localdata)
-
- taskdata = bb.taskdata.TaskData(self.configuration.abort)
- taskdata.add_provider(localdata, self.recipecache, pkgs_to_build[0])
- taskdata.add_unresolved(localdata, self.recipecache)
+ runlist, taskdata = self.prepareTreeData(pkgs_to_build, None)
targetid = taskdata.getbuild_id(pkgs_to_build[0])
fnid = taskdata.build_targets[targetid][0]
--
1.8.1.2
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 09/13] bitbake: event: do not abort on UI handlers error
2013-09-09 16:40 [PATCH 00/13] webhob related changes in bitbake Alex DAMIAN
` (7 preceding siblings ...)
2013-09-09 16:40 ` [PATCH 08/13] bitbake: cooker: use prepareTreeData when computing environment Alex DAMIAN
@ 2013-09-09 16:41 ` Alex DAMIAN
2013-09-09 16:49 ` Richard Purdie
2013-09-09 16:41 ` [PATCH 10/13] bitbake: add SinglePackageInfo event Alex DAMIAN
` (3 subsequent siblings)
12 siblings, 1 reply; 18+ messages in thread
From: Alex DAMIAN @ 2013-09-09 16:41 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Ignore runtime errors if UI handlers do something
funny, as exiting while the loop is running.
This allows the builder to continue even if something
happens to the UI handlers, useful for remote bitbake
server.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/event.py | 32 ++++++++++++++++++--------------
1 file changed, 18 insertions(+), 14 deletions(-)
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
index 67cfcea..1258471 100644
--- a/bitbake/lib/bb/event.py
+++ b/bitbake/lib/bb/event.py
@@ -133,20 +133,24 @@ def fire_ui_handlers(event, d):
return
errors = []
- for h in _ui_handlers:
- #print "Sending event %s" % event
- try:
- if not _ui_logfilters[h].filter(event):
- continue
- # We use pickle here since it better handles object instances
- # which xmlrpc's marshaller does not. Events *must* be serializable
- # by pickle.
- if hasattr(_ui_handlers[h].event, "sendpickle"):
- _ui_handlers[h].event.sendpickle((pickle.dumps(event)))
- else:
- _ui_handlers[h].event.send(event)
- except:
- errors.append(h)
+ try:
+ for h in _ui_handlers:
+ #print "Sending event %s" % event
+ try:
+ if not _ui_logfilters[h].filter(event):
+ continue
+ # We use pickle here since it better handles object instances
+ # which xmlrpc's marshaller does not. Events *must* be serializable
+ # by pickle.
+ if hasattr(_ui_handlers[h].event, "sendpickle"):
+ _ui_handlers[h].event.sendpickle((pickle.dumps(event)))
+ else:
+ _ui_handlers[h].event.send(event)
+ except:
+ errors.append(h)
+ except RuntimeError: # may happen that the ui handler simple disappears :D
+ errors.append(h)
+
for h in errors:
del _ui_handlers[h]
--
1.8.1.2
^ permalink raw reply related [flat|nested] 18+ messages in thread* Re: [PATCH 09/13] bitbake: event: do not abort on UI handlers error
2013-09-09 16:41 ` [PATCH 09/13] bitbake: event: do not abort on UI handlers error Alex DAMIAN
@ 2013-09-09 16:49 ` Richard Purdie
0 siblings, 0 replies; 18+ messages in thread
From: Richard Purdie @ 2013-09-09 16:49 UTC (permalink / raw)
To: Alex DAMIAN; +Cc: bitbake-devel
On Mon, 2013-09-09 at 17:41 +0100, Alex DAMIAN wrote:
> From: Alexandru DAMIAN <alexandru.damian@intel.com>
>
> Ignore runtime errors if UI handlers do something
> funny, as exiting while the loop is running.
This does look rather strange to me. How can a handler exit within the
loop? The handler is either another process or something we're using an
xmlrpc call too. Fine, the other end can disappear but our local end
(and entry in _ui_handlers) should exist until we remove it ourselves?
Only the server itself can do the removal and the server is single
threaded?
Cheers,
Richard
> This allows the builder to continue even if something
> happens to the UI handlers, useful for remote bitbake
> server.
>
> Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
> ---
> bitbake/lib/bb/event.py | 32 ++++++++++++++++++--------------
> 1 file changed, 18 insertions(+), 14 deletions(-)
>
> diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
> index 67cfcea..1258471 100644
> --- a/bitbake/lib/bb/event.py
> +++ b/bitbake/lib/bb/event.py
> @@ -133,20 +133,24 @@ def fire_ui_handlers(event, d):
> return
>
> errors = []
> - for h in _ui_handlers:
> - #print "Sending event %s" % event
> - try:
> - if not _ui_logfilters[h].filter(event):
> - continue
> - # We use pickle here since it better handles object instances
> - # which xmlrpc's marshaller does not. Events *must* be serializable
> - # by pickle.
> - if hasattr(_ui_handlers[h].event, "sendpickle"):
> - _ui_handlers[h].event.sendpickle((pickle.dumps(event)))
> - else:
> - _ui_handlers[h].event.send(event)
> - except:
> - errors.append(h)
> + try:
> + for h in _ui_handlers:
> + #print "Sending event %s" % event
> + try:
> + if not _ui_logfilters[h].filter(event):
> + continue
> + # We use pickle here since it better handles object instances
> + # which xmlrpc's marshaller does not. Events *must* be serializable
> + # by pickle.
> + if hasattr(_ui_handlers[h].event, "sendpickle"):
> + _ui_handlers[h].event.sendpickle((pickle.dumps(event)))
> + else:
> + _ui_handlers[h].event.send(event)
> + except:
> + errors.append(h)
> + except RuntimeError: # may happen that the ui handler simple disappears :D
> + errors.append(h)
> +
> for h in errors:
> del _ui_handlers[h]
>
^ permalink raw reply [flat|nested] 18+ messages in thread
* [PATCH 10/13] bitbake: add SinglePackageInfo event
2013-09-09 16:40 [PATCH 00/13] webhob related changes in bitbake Alex DAMIAN
` (8 preceding siblings ...)
2013-09-09 16:41 ` [PATCH 09/13] bitbake: event: do not abort on UI handlers error Alex DAMIAN
@ 2013-09-09 16:41 ` Alex DAMIAN
2013-09-09 16:41 ` [PATCH 11/13] bitbake: event: adding an event to pass build file sizes Alex DAMIAN
` (2 subsequent siblings)
12 siblings, 0 replies; 18+ messages in thread
From: Alex DAMIAN @ 2013-09-09 16:41 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
We add a new event that will be triggered by the
package-building class, containing the information
about the currently built package.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/event.py | 8 ++++++++
bitbake/lib/bb/ui/knotty.py | 1 +
2 files changed, 9 insertions(+)
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
index 1258471..4be0e5b 100644
--- a/bitbake/lib/bb/event.py
+++ b/bitbake/lib/bb/event.py
@@ -593,6 +593,14 @@ class PackageInfo(Event):
Event.__init__(self)
self._pkginfolist = pkginfolist
+class SinglePackageInfo(Event):
+ """
+ Single Package Information sent on emit_pkgdata
+ """
+ def __init__(self, data):
+ Event.__init__(self)
+ self._data = data
+
class SanityCheck(Event):
"""
Event to issue sanity check
diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index 4520046..010530a 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -473,6 +473,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
# ignore
if isinstance(event, (bb.event.BuildBase,
+ bb.event.SinglePackageInfo,
bb.event.StampUpdate,
bb.event.ConfigParsed,
bb.event.RecipeParsed,
--
1.8.1.2
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 11/13] bitbake: event: adding an event to pass build file sizes
2013-09-09 16:40 [PATCH 00/13] webhob related changes in bitbake Alex DAMIAN
` (9 preceding siblings ...)
2013-09-09 16:41 ` [PATCH 10/13] bitbake: add SinglePackageInfo event Alex DAMIAN
@ 2013-09-09 16:41 ` Alex DAMIAN
2013-09-09 16:41 ` [PATCH 12/13] bitbake: xmlrpc: delete unused code BitBakeUIEventServer Alex DAMIAN
2013-09-09 16:41 ` [PATCH 13/13] bitbake: cooker: add extra recipe information Alex DAMIAN
12 siblings, 0 replies; 18+ messages in thread
From: Alex DAMIAN @ 2013-09-09 16:41 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Adding an event, ignored by default, that will pass
information about the content of each package file that
gets built.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/event.py | 8 ++++++++
bitbake/lib/bb/ui/knotty.py | 1 +
2 files changed, 9 insertions(+)
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
index 4be0e5b..781b8e7 100644
--- a/bitbake/lib/bb/event.py
+++ b/bitbake/lib/bb/event.py
@@ -601,6 +601,14 @@ class SinglePackageInfo(Event):
Event.__init__(self)
self._data = data
+class PackageFileSizes(Event):
+ """
+ Event that contains information about the file sizes contained in a package.
+ """
+ def __init__(self, data):
+ Event.__init__(self)
+ self._data = data
+
class SanityCheck(Event):
"""
Event to issue sanity check
diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index 010530a..1f09a55 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -473,6 +473,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
# ignore
if isinstance(event, (bb.event.BuildBase,
+ bb.event.PackageFileSizes,
bb.event.SinglePackageInfo,
bb.event.StampUpdate,
bb.event.ConfigParsed,
--
1.8.1.2
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 12/13] bitbake: xmlrpc: delete unused code BitBakeUIEventServer
2013-09-09 16:40 [PATCH 00/13] webhob related changes in bitbake Alex DAMIAN
` (10 preceding siblings ...)
2013-09-09 16:41 ` [PATCH 11/13] bitbake: event: adding an event to pass build file sizes Alex DAMIAN
@ 2013-09-09 16:41 ` Alex DAMIAN
2013-09-09 16:41 ` [PATCH 13/13] bitbake: cooker: add extra recipe information Alex DAMIAN
12 siblings, 0 replies; 18+ messages in thread
From: Alex DAMIAN @ 2013-09-09 16:41 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
BitBakeUIEventServer is an unused class that pushes UI
events over a separate thread.
The current version of XMLRPC server works just fine with
the classic UI event handlers, so this class is not needed.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/server/xmlrpc.py | 45 -----------------------------------------
1 file changed, 45 deletions(-)
diff --git a/bitbake/lib/bb/server/xmlrpc.py b/bitbake/lib/bb/server/xmlrpc.py
index a28672c..8326623 100644
--- a/bitbake/lib/bb/server/xmlrpc.py
+++ b/bitbake/lib/bb/server/xmlrpc.py
@@ -169,51 +169,6 @@ class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
self.end_headers()
self.wfile.write(response)
-class BitBakeUIEventServer(threading.Thread):
- class EventAdapter():
- """
- Adapter to wrap our event queue since the caller (bb.event) expects to
- call a send() method, but our actual queue only has put()
- """
- def __init__(self, notify):
- self.queue = []
- self.notify = notify
- self.qlock = threading.Lock()
-
- def send(self, event):
- self.qlock.acquire()
- self.queue.append(event)
- self.qlock.release()
- self.notify.set()
-
- def get(self):
- self.qlock.acquire()
- if len(self.queue) == 0:
- self.qlock.release()
- return None
- e = self.queue.pop(0)
- if len(self.queue) == 0:
- self.notify.clear()
- self.qlock.release()
- return e
-
- def __init__(self, connection):
- self.connection = connection
- self.notify = threading.Event()
- self.event = BitBakeUIEventServer.EventAdapter(self.notify)
- self.quit = False
- threading.Thread.__init__(self)
-
- def terminateServer(self):
- self.quit = True
-
- def run(self):
- while not self.quit:
- self.notify.wait(0.1)
- evt = self.event.get()
- if evt:
- self.connection.event.sendpickle(pickle.dumps(evt))
-
class XMLRPCProxyServer(BaseImplServer):
""" not a real working server, but a stub for a proxy server connection
--
1.8.1.2
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 13/13] bitbake: cooker: add extra recipe information
2013-09-09 16:40 [PATCH 00/13] webhob related changes in bitbake Alex DAMIAN
` (11 preceding siblings ...)
2013-09-09 16:41 ` [PATCH 12/13] bitbake: xmlrpc: delete unused code BitBakeUIEventServer Alex DAMIAN
@ 2013-09-09 16:41 ` Alex DAMIAN
12 siblings, 0 replies; 18+ messages in thread
From: Alex DAMIAN @ 2013-09-09 16:41 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Adding in the extra recipe information when creating
the dependency tree information.
This works in server-mode because the HOB extra_cache
is already enabled. In normal mode, it will do nothing
with no performance impact.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/cooker.py | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 1eb1768..bad930b 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -439,6 +439,29 @@ class BBCooker:
depend_tree["pn"][pn] = {}
depend_tree["pn"][pn]["filename"] = fn
depend_tree["pn"][pn]["version"] = version
+
+ # This data is needed for webhob;
+ # it's here if the bitbake runs in server mode
+ try:
+ summary = self.recipecache.summary[fn]
+ lic = self.recipecache.license[fn]
+ section = self.recipecache.section[fn]
+ description = self.recipecache.description[fn]
+ homepage = self.recipecache.homepage[fn]
+ bugtracker = self.recipecache.bugtracker[fn]
+ inherits = self.recipecache.inherits.get(fn, None)
+ depend_tree["pn"][pn]["filename"] = fn
+ depend_tree["pn"][pn]["version"] = version
+ depend_tree["pn"][pn]["summary"] = summary
+ depend_tree["pn"][pn]["license"] = lic
+ depend_tree["pn"][pn]["section"] = section
+ depend_tree["pn"][pn]["description"] = description
+ depend_tree["pn"][pn]["inherits"] = inherits
+ depend_tree["pn"][pn]["homepage"] = homepage
+ depend_tree["pn"][pn]["bugtracker"] = bugtracker
+ except KeyError:
+ pass
+
for dep in rq.rqdata.runq_depends[task]:
depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
deppn = self.recipecache.pkg_fn[depfn]
--
1.8.1.2
^ permalink raw reply related [flat|nested] 18+ messages in thread