[PATCH 0/2] bitbake patches

[PATCH 0/2] bitbake patches

[Paul Eggleton]

A couple of bitbake patches.

The patches (against Poky, but apply cleanly with -p2 against bitbake
master) are available in the git repository at:
  git://git.yoctoproject.org/poky-contrib paule/bitbake-errors4
  http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/bitbake-errors4

Paul Eggleton (2):
  bitbake: fix setscene task start/failure handling
  bitbake/runqueue: make dry-run do everything except execute

 bitbake/lib/bb/build.py       |   10 +++++++++-
 bitbake/lib/bb/runqueue.py    |   27 +++++++++++++++------------
 bitbake/lib/bb/ui/knotty.py   |    9 +++++++++
 bitbake/lib/bb/ui/uihelper.py |    3 +--
 4 files changed, 34 insertions(+), 15 deletions(-)

-- 
1.7.5.4

[PATCH 1/2] bitbake: fix setscene task start/failure handling

[Paul Eggleton]

* When a setscene task starts, print out that it's starting in the UI
  (ensuring we get the correct task name)
* When a setscene task fails, ensure we remove it from the list of
  running tasks so that if you break out any time afterwards it
  is not still listed.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 bitbake/lib/bb/build.py       |   10 +++++++++-
 bitbake/lib/bb/runqueue.py    |   18 +++++++++++++-----
 bitbake/lib/bb/ui/knotty.py   |    9 +++++++++
 bitbake/lib/bb/ui/uihelper.py |    3 +--
 4 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index e85d7c4..b7031ab 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -96,6 +96,12 @@ class TaskFailed(TaskBase):
         self.errprinted = errprinted
         super(TaskFailed, self).__init__(task, metadata)
 
+class TaskFailedSilent(TaskBase):
+    """Task execution failed (silently)"""
+    def __init__(self, task, logfile, metadata):
+        self.logfile = logfile
+        super(TaskFailedSilent, self).__init__(task, metadata)
+
 class TaskInvalid(TaskBase):
 
     def __init__(self, task, metadata):
@@ -334,7 +340,9 @@ def _exec_task(fn, task, d, quieterr):
         for func in (postfuncs or '').split():
             exec_func(func, localdata)
     except FuncFailed as exc:
-        if not quieterr:
+        if quieterr:
+            event.fire(TaskFailedSilent(task, logfn, localdata), localdata)
+        else:
             errprinted = errchk.triggered
             logger.error(str(exc))
             event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata)
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index f08f93a..0652c49 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -1729,6 +1729,15 @@ class runQueueEvent(bb.event.Event):
         self.stats = stats.copy()
         bb.event.Event.__init__(self)
 
+class sceneQueueEvent(runQueueEvent):
+    """
+    Base sceneQueue event class
+    """
+    def __init__(self, task, stats, rq, noexec=False):
+        runQueueEvent.__init__(self, task, stats, rq)
+        realtask = rq.rqdata.runq_setscene[task]
+        self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
+
 class runQueueTaskStarted(runQueueEvent):
     """
     Event notifing a task was started
@@ -1737,12 +1746,12 @@ class runQueueTaskStarted(runQueueEvent):
         runQueueEvent.__init__(self, task, stats, rq)
         self.noexec = noexec
 
-class sceneQueueTaskStarted(runQueueEvent):
+class sceneQueueTaskStarted(sceneQueueEvent):
     """
     Event notifing a setscene task was started
     """
     def __init__(self, task, stats, rq, noexec=False):
-        runQueueEvent.__init__(self, task, stats, rq)
+        sceneQueueEvent.__init__(self, task, stats, rq)
         self.noexec = noexec
 
 class runQueueTaskFailed(runQueueEvent):
@@ -1753,14 +1762,13 @@ class runQueueTaskFailed(runQueueEvent):
         runQueueEvent.__init__(self, task, stats, rq)
         self.exitcode = exitcode
 
-class sceneQueueTaskFailed(runQueueEvent):
+class sceneQueueTaskFailed(sceneQueueEvent):
     """
     Event notifing a setscene task failed
     """
     def __init__(self, task, stats, exitcode, rq):
-        runQueueEvent.__init__(self, task, stats, rq)
+        sceneQueueEvent.__init__(self, task, stats, rq)
         self.exitcode = exitcode
-        self.taskstring = rq.rqdata.get_user_idstring(task, "_setscene")
 
 class runQueueTaskCompleted(runQueueEvent):
     """
diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index e2e6ac3..14989d4 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -238,6 +238,10 @@ def main(server, eventHandler):
                         logger.error("%s", reason)
                 continue
 
+            if isinstance(event, bb.runqueue.sceneQueueTaskStarted):
+                logger.info("Running setscene task %d of %d (%s)" % (event.stats.completed + event.stats.active + event.stats.failed + 1, event.stats.total, event.taskstring))
+                continue
+
             if isinstance(event, bb.runqueue.runQueueTaskStarted):
                 if event.noexec:
                     tasktype = 'noexec task'
@@ -256,6 +260,11 @@ def main(server, eventHandler):
                              event.taskid, event.taskstring, event.exitcode)
                 continue
 
+            if isinstance(event, bb.runqueue.sceneQueueTaskFailed):
+                logger.warn("Setscene task %s (%s) failed with exit code '%s' - real task will be run instead",
+                             event.taskid, event.taskstring, event.exitcode)
+                continue
+
             # ignore
             if isinstance(event, (bb.event.BuildBase,
                                   bb.event.StampUpdate,
diff --git a/bitbake/lib/bb/ui/uihelper.py b/bitbake/lib/bb/ui/uihelper.py
index bbf5135..4116dab 100644
--- a/bitbake/lib/bb/ui/uihelper.py
+++ b/bitbake/lib/bb/ui/uihelper.py
@@ -32,9 +32,8 @@ class BBUIHelper:
         if isinstance(event, bb.build.TaskSucceeded):
             del self.running_tasks[event.pid]
             self.needUpdate = True
-        if isinstance(event, bb.build.TaskFailed):
+        if isinstance(event, bb.build.TaskFailed) or isinstance(event, bb.build.TaskFailedSilent):
             del self.running_tasks[event.pid]
-            self.failed_tasks.append( { 'title' : "%s %s" % (event._package, event._task)})
             self.needUpdate = True
 
     def getTasks(self):
-- 
1.7.5.4

Re: [PATCH 1/2] bitbake: fix setscene task start/failure handling

[McClintock Matthew-B29882]

On Mon, Feb 27, 2012 at 12:54 PM, Paul Eggleton
<paul.eggleton@linux.intel.com> wrote:
> +            if isinstance(event, bb.runqueue.sceneQueueTaskFailed):
> +                logger.warn("Setscene task %s (%s) failed with exit code '%s' - real task will be run instead",
> +                             event.taskid, event.taskstring, event.exitcode)

This removes the "ERROR:" message, right?

-M

Re: [PATCH 1/2] bitbake: fix setscene task start/failure handling

[Paul Eggleton]

On Monday 27 February 2012 19:15:46 McClintock Matthew-B29882 wrote:
> On Mon, Feb 27, 2012 at 12:54 PM, Paul Eggleton
> 
> <paul.eggleton@linux.intel.com> wrote:
> > +            if isinstance(event, bb.runqueue.sceneQueueTaskFailed):
> > +                logger.warn("Setscene task %s (%s) failed with exit code
> > '%s' - real task will be run instead", +                            
> > event.taskid, event.taskstring, event.exitcode)
> This removes the "ERROR:" message, right?

If a failure logs an ERROR itself then no, this would still be printed. 
However, in the case of an error situation such as one I simulated (unable to 
read from the sstate-cache file) nothing was being printed at all, which is not 
particularly helpful. Thus this part of the change.

Cheers,
Paull

-- 

Paul Eggleton
Intel Open Source Technology Centre

Re: [PATCH 1/2] bitbake: fix setscene task start/failure handling

[McClintock Matthew-B29882]

On Mon, Feb 27, 2012 at 5:45 PM, Paul Eggleton
<paul.eggleton@linux.intel.com> wrote:
> On Monday 27 February 2012 19:15:46 McClintock Matthew-B29882 wrote:
>> On Mon, Feb 27, 2012 at 12:54 PM, Paul Eggleton
>>
>> <paul.eggleton@linux.intel.com> wrote:
>> > +            if isinstance(event, bb.runqueue.sceneQueueTaskFailed):
>> > +                logger.warn("Setscene task %s (%s) failed with exit code
>> > '%s' - real task will be run instead", +
>> > event.taskid, event.taskstring, event.exitcode)
>> This removes the "ERROR:" message, right?
>
> If a failure logs an ERROR itself then no, this would still be printed.
> However, in the case of an error situation such as one I simulated (unable to
> read from the sstate-cache file) nothing was being printed at all, which is not
> particularly helpful. Thus this part of the change.

I was referring to when the setscene tasks fails and we emit an ERROR
when it's not really a fatal error as we can just rerun the real task.
It's still an "issue" - but when I check my build logs for issues I
grep for ERROR and I'd rather leave ERROR for FATAL ERRORS.

Just my 2 cents.

-M

Re: [PATCH 1/2] bitbake: fix setscene task start/failure handling

[Paul Eggleton]

On Tuesday 28 February 2012 16:32:29 McClintock Matthew-B29882 wrote:
> I was referring to when the setscene tasks fails and we emit an ERROR
> when it's not really a fatal error as we can just rerun the real task.
> It's still an "issue" - but when I check my build logs for issues I
> grep for ERROR and I'd rather leave ERROR for FATAL ERRORS.

I agree and I'm happy to look at that as a separate issue if it is indeed 
still occurring in master; my immediate concern was not getting any message at 
all though.

Cheers,
Paul

-- 

Paul Eggleton
Intel Open Source Technology Centre

[PATCH 2/2] bitbake/runqueue: make dry-run do everything except execute

[Paul Eggleton]

Make dry-run do everything except executing the task, instead of
cutting it off earlier. This fully tests the code path for running the
child task (parsing and fakeroot), as well as enabling future
functionality such as using dry-run to produce signature files.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 bitbake/lib/bb/runqueue.py |    9 ++-------
 1 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 0652c49..d4b2cd4 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -1187,7 +1187,8 @@ class RunQueueExecute:
                     logger.critical(str(exc))
                 os._exit(1)
             try:
-                ret = bb.build.exec_task(fn, taskname, the_data)
+                if not self.cooker.configuration.dry_run:
+                    ret = bb.build.exec_task(fn, taskname, the_data)
                 os._exit(ret)
             except:
                 os._exit(1)
@@ -1370,12 +1371,6 @@ class RunQueueExecuteTasks(RunQueueExecute):
                                 self.rqdata.get_user_idstring(task))
                 self.task_skip(task)
                 return True
-            elif self.cooker.configuration.dry_run:
-                self.runq_running[task] = 1
-                self.runq_buildable[task] = 1
-                self.stats.taskActive()
-                self.task_complete(task)
-                return True
 
             taskdep = self.rqdata.dataCache.task_deps[fn]
             if 'noexec' in taskdep and taskname in taskdep['noexec']:
-- 
1.7.5.4