[PATCH 0/6] Switch hob from buildFiles to buildTargets

[PATCH 0/6] Switch hob from buildFiles to buildTargets

[Joshua Lock]

The initial hob merge came with the caveat that it used buildFiles and
therefore a build in a new builddir wouldn't have the required native tools
to actually generate the rootfs.

The key patch in this series switches to using buildTargets by injecting the
directory the generated image recipe is saved to into BBPATH and a BBFILES
pattern then reparsing the files before building.

Further this series includes a bug fix or two and a patch that got lost in
a branch somewhere.

The following changes since commit 5c8eeefc79455f058dda8f04cf4c12dc5418e00f:

  cooker: only return *Found events if something was actually found (2011-07-14 15:12:40 +0100)

are available in the git repository at:
  git://github.com/incandescant/bitbake hob
  https://github.com/incandescant/bitbake/tree/hob

Joshua Lock (6):
  ui/hob: replace the ugly static command map
  ui/crumbs/hobprefs: add missing import
  command|cooker: Add reparseFiles command
  ui/crumbs/hobeventhandler: reparse files before running other
    commands
  ui/crumbs/tasklistmodel: fix saving recipes
  ui/hob: switch from buildFile to buildTargets for custom image builds

 lib/bb/command.py                   |    8 ++
 lib/bb/cooker.py                    |   54 ++++++++-----
 lib/bb/ui/crumbs/configurator.py    |   18 ++++
 lib/bb/ui/crumbs/hobeventhandler.py |  147 +++++++++++++++++++++--------------
 lib/bb/ui/crumbs/hobprefs.py        |    1 +
 lib/bb/ui/crumbs/tasklistmodel.py   |    3 +-
 lib/bb/ui/hob.py                    |   26 +++----
 7 files changed, 161 insertions(+), 96 deletions(-)

-- 
1.7.6

[PATCH 1/6] ui/hob: replace the ugly static command map

[Joshua Lock]

The command_map was never a good idea, what's implemented here is a
fraction less ugly but a significant factor more readable and therefore
easy to maintain.
The method implemented in this patch also has the advantage of not being
static meaning we can determine the desired runCommand arguments
dynamically at call time.

Signed-off-by: Joshua Lock <josh@linux.intel.com>
---
 lib/bb/ui/crumbs/hobeventhandler.py |   63 ++++++++++++++++++++--------------
 lib/bb/ui/hob.py                    |    2 +-
 2 files changed, 38 insertions(+), 27 deletions(-)

diff --git a/lib/bb/ui/crumbs/hobeventhandler.py b/lib/bb/ui/crumbs/hobeventhandler.py
index fa79e0c..50f3419 100644
--- a/lib/bb/ui/crumbs/hobeventhandler.py
+++ b/lib/bb/ui/crumbs/hobeventhandler.py
@@ -61,8 +61,11 @@ class HobHandler(gobject.GObject):
                                    gobject.TYPE_STRING)),
     }
 
+    (CFG_PATH_LOCAL, CFG_PATH_HOB, CFG_PATH_LAYERS, CFG_FILES_DISTRO, CFG_FILES_MACH, CFG_FILES_SDK, FILES_MATCH_CLASS, GENERATE_TGTS) = range(8)
+
     def __init__(self, taskmodel, server):
         gobject.GObject.__init__(self)
+
         self.current_command = None
         self.building = None
         self.gplv3_excluded = False
@@ -74,30 +77,40 @@ class HobHandler(gobject.GObject):
         self.model = taskmodel
         self.server = server
 
-        self.command_map = {
-            "findConfigFilePathLocal" : ("findConfigFilePath", ["hob.local.conf"], "findConfigFilePathHobLocal"),
-            "findConfigFilePathHobLocal" : ("findConfigFilePath", ["bblayers.conf"], "findConfigFilePathLayers"),
-            "findConfigFilePathLayers" : ("findConfigFiles", ["DISTRO"], "findConfigFilesDistro"),
-            "findConfigFilesDistro" : ("findConfigFiles", ["MACHINE"], "findConfigFilesMachine"),
-            "findConfigFilesMachine" : ("findConfigFiles", ["MACHINE-SDK"], "findConfigFilesSdkMachine"),
-            "findConfigFilesSdkMachine" : ("findFilesMatchingInDir", ["rootfs_", "classes"], "findFilesMatchingPackage"),
-            "findFilesMatchingPackage" : ("generateTargetsTree", ["classes/image.bbclass"], None),
-            "generateTargetsTree"  : (None, [], None),
-            }
-
     def run_next_command(self):
-        # FIXME: this is ugly and I *will* replace it
-        if self.current_command:
-            if not self.generating:
-                self.emit("generating-data")
-                self.generating = True
-            next_cmd = self.command_map[self.current_command]
-            command = next_cmd[0]
-            argument = next_cmd[1]
-            self.current_command = next_cmd[2]
-            args = [command]
-            args.extend(argument)
-            self.server.runCommand(args)
+        if self.current_command and not self.generating:
+            self.emit("generating-data")
+            self.generating = True
+
+        if self.current_command == self.CFG_PATH_LOCAL:
+            self.current_command = self.CFG_PATH_HOB
+            self.server.runCommand(["findConfigFilePath", "hob.local.conf"])
+        elif self.current_command == self.CFG_PATH_HOB:
+            self.current_command = self.CFG_PATH_LAYERS
+            self.server.runCommand(["findConfigFilePath", "bblayers.conf"])
+        elif self.current_command == self.CFG_PATH_LAYERS:
+            self.current_command = self.CFG_FILES_DISTRO
+            self.server.runCommand(["findConfigFiles", "DISTRO"])
+        elif self.current_command == self.CFG_FILES_DISTRO:
+            self.current_command = self.CFG_FILES_MACH
+            self.server.runCommand(["findConfigFiles", "MACHINE"])
+        elif self.current_command == self.CFG_FILES_MACH:
+            self.current_command = self.CFG_FILES_SDK
+            self.server.runCommand(["findConfigFiles", "MACHINE-SDK"])
+        elif self.current_command == self.CFG_FILES_SDK:
+            self.current_command = self.FILES_MATCH_CLASS
+            self.server.runCommand(["findFilesMatchingInDir", "rootfs_", "classes"])
+        elif self.current_command == self.FILES_MATCH_CLASS:
+            self.current_command = self.GENERATE_TGTS
+            self.server.runCommand(["generateTargetsTree", "classes/image.bbclass"])
+        elif self.current_command == self.GENERATE_TGTS:
+            if self.generating:
+                self.emit("data-generated")
+                self.generating = False
+            self.current_command = None
+        else:
+            # No command?
+            pass
 
     def handle_event(self, event, running_build, pbar):
         if not event:
@@ -107,8 +120,6 @@ class HobHandler(gobject.GObject):
         if self.building:
             running_build.handle_event(event)
         elif isinstance(event, bb.event.TargetsTreeGenerated):
-            self.emit("data-generated")
-            self.generating = False
             if event._model:
                 self.model.populate(event._model)
         elif isinstance(event, bb.event.ConfigFilesFound):
@@ -186,7 +197,7 @@ class HobHandler(gobject.GObject):
         selected_packages, _ = self.model.get_selected_packages()
         self.emit("reload-triggered", img, " ".join(selected_packages))
         self.server.runCommand(["reparseFiles"])
-        self.current_command = "findConfigFilePathLayers"
+        self.current_command = self.CFG_PATH_LAYERS
         self.run_next_command()
 
     def set_bbthreads(self, threads):
diff --git a/lib/bb/ui/hob.py b/lib/bb/ui/hob.py
index 06d936e..888f42e 100644
--- a/lib/bb/ui/hob.py
+++ b/lib/bb/ui/hob.py
@@ -906,7 +906,7 @@ def main (server, eventHandler):
 
     try:
         # kick the while thing off
-        handler.current_command = "findConfigFilePathLocal"
+        handler.current_command = handler.CFG_PATH_LOCAL
         server.runCommand(["findConfigFilePath", "local.conf"])
     except xmlrpclib.Fault:
         print("XMLRPC Fault getting commandline:\n %s" % x)
-- 
1.7.6

[PATCH 2/6] ui/crumbs/hobprefs: add missing import

[Joshua Lock]

glib.idle_add is used so the glib module must be imported

Fixes [YOCTO #1248]

Signed-off-by: Joshua Lock <josh@linux.intel.com>
---
 lib/bb/ui/crumbs/hobprefs.py |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diff --git a/lib/bb/ui/crumbs/hobprefs.py b/lib/bb/ui/crumbs/hobprefs.py
index f186410..779c14a 100644
--- a/lib/bb/ui/crumbs/hobprefs.py
+++ b/lib/bb/ui/crumbs/hobprefs.py
@@ -19,6 +19,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import gtk
+import glib
 from bb.ui.crumbs.configurator import Configurator
 
 class HobPrefs(gtk.Dialog):
-- 
1.7.6

[PATCH 3/6] command|cooker: Add reparseFiles command

[Joshua Lock]

Add command reparseFiles to reparse bb files in a running cooker instance.

Fixes [YOCTO #1249]

Signed-off-by: Joshua Lock <josh@linux.intel.com>
---
 lib/bb/command.py |    8 +++++++
 lib/bb/cooker.py  |   54 +++++++++++++++++++++++++++++++++-------------------
 2 files changed, 42 insertions(+), 20 deletions(-)

diff --git a/lib/bb/command.py b/lib/bb/command.py
index a902da2..893a6d9 100644
--- a/lib/bb/command.py
+++ b/lib/bb/command.py
@@ -311,6 +311,14 @@ class CommandsAsync:
         command.finishAsyncCommand()
     parseFiles.needcache = True
 
+    def reparseFiles(self, command, params):
+        """
+        Reparse .bb files
+        """
+        command.cooker.reparseFiles()
+        command.finishAsyncCommand()
+    reparseFiles.needcache = True
+
     def compareRevisions(self, command, params):
         """
         Parse the .bb files
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 23f2c76..e4daaef 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -122,21 +122,8 @@ class BBCooker:
                 logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc))
                 sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name)
 
-        self.configuration.data = bb.data.init()
-
-        bb.data.inheritFromOS(self.configuration.data)
-
-        try:
-            self.parseConfigurationFiles(self.configuration.prefile,
-                                         self.configuration.postfile)
-        except SyntaxError:
-            sys.exit(1)
-        except Exception:
-            logger.exception("Error parsing configuration files")
-            sys.exit(1)
-
-        if not self.configuration.cmd:
-            self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build"
+        self.configuration.data = None
+        self.loadConfigurationData()
 
         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True)
         if bbpkgs and len(self.configuration.pkgs_to_build) == 0:
@@ -163,6 +150,26 @@ class BBCooker:
 
         self.parser = None
 
+    def loadConfigurationData(self):
+        self.configuration.data = bb.data.init()
+
+        if not self.server_registration_cb:
+            bb.data.setVar("BB_WORKERCONTEXT", "1", self.configuration.data)
+
+        bb.data.inheritFromOS(self.configuration.data)
+
+        try:
+            self.parseConfigurationFiles(self.configuration.prefile,
+                                         self.configuration.postfile)
+        except SyntaxError:
+            sys.exit(1)
+        except Exception:
+            logger.exception("Error parsing configuration files")
+            sys.exit(1)
+
+        if not self.configuration.cmd:
+            self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build"
+
     def parseConfiguration(self):
 
 
@@ -508,7 +515,7 @@ class BBCooker:
         bb.data.expandKeys(localdata)
 
         # Handle PREFERRED_PROVIDERS
-        for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
+        for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, True) or "").split():
             try:
                 (providee, provider) = p.split(':')
             except:
@@ -999,8 +1006,8 @@ class BBCooker:
 
         self.server_registration_cb(buildTargetsIdle, rq)
 
-    def updateCache(self):
-        if self.state == state.running:
+    def updateCache(self, force=False):
+        if self.state == state.running and not force:
             return
 
         if self.state in (state.shutdown, state.stop):
@@ -1010,6 +1017,8 @@ class BBCooker:
         if self.state != state.parsing:
             self.parseConfiguration ()
 
+            if self.status:
+                del self.status
             self.status = bb.cache.CacheData(self.caches_array)
 
             ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
@@ -1083,7 +1092,7 @@ class BBCooker:
 
         collectlog.debug(1, "collecting .bb files")
 
-        files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
+        files = (data.getVar( "BBFILES", self.configuration.data, True) or "").split()
         data.setVar("BBFILES", " ".join(files), self.configuration.data)
 
         # Sort files by priority
@@ -1140,7 +1149,8 @@ class BBCooker:
             base = os.path.basename(f).replace('.bbappend', '.bb')
             if not base in self.appendlist:
                self.appendlist[base] = []
-            self.appendlist[base].append(f)
+            if f not in self.appendlist[base]:
+                self.appendlist[base].append(f)
 
         # Find overlayed recipes
         # bbfiles will be in priority order which makes this easy
@@ -1181,6 +1191,10 @@ class BBCooker:
     def stop(self):
         self.state = state.stop
 
+    def reparseFiles(self):
+        self.loadConfigurationData()
+        self.updateCache(force=True)
+
 def server_main(cooker, func, *args):
     cooker.pre_serve()
 
-- 
1.7.6

Re: [PATCH 3/6] command|cooker: Add reparseFiles command

[Richard Purdie]

On Fri, 2011-07-15 at 13:20 -0700, Joshua Lock wrote:
> Add command reparseFiles to reparse bb files in a running cooker instance.
> 
> Fixes [YOCTO #1249]
> 
> Signed-off-by: Joshua Lock <josh@linux.intel.com>
> ---
>  lib/bb/command.py |    8 +++++++
>  lib/bb/cooker.py  |   54 +++++++++++++++++++++++++++++++++-------------------
>  2 files changed, 42 insertions(+), 20 deletions(-)
> 

> diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
> index 23f2c76..e4daaef 100644
> --- a/lib/bb/cooker.py
> +++ b/lib/bb/cooker.py
> @@ -122,21 +122,8 @@ class BBCooker:
>                  logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc))
>                  sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name)
>  
> -        self.configuration.data = bb.data.init()
> -
> -        bb.data.inheritFromOS(self.configuration.data)
> -
> -        try:
> -            self.parseConfigurationFiles(self.configuration.prefile,
> -                                         self.configuration.postfile)
> -        except SyntaxError:
> -            sys.exit(1)
> -        except Exception:
> -            logger.exception("Error parsing configuration files")
> -            sys.exit(1)
> -
> -        if not self.configuration.cmd:
> -            self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build"
> +        self.configuration.data = None
> +        self.loadConfigurationData()
>  
>          bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True)
>          if bbpkgs and len(self.configuration.pkgs_to_build) == 0:
> @@ -163,6 +150,26 @@ class BBCooker:
>  
>          self.parser = None
>  
> +    def loadConfigurationData(self):
> +        self.configuration.data = bb.data.init()
> +
> +        if not self.server_registration_cb:
> +            bb.data.setVar("BB_WORKERCONTEXT", "1", self.configuration.data)

This looks like one of the last remaining differences between the
bitbake in poky and the upstream. See:

http://git.openembedded.net/cgit.cgi/bitbake/commit/?h=master-poky&id=a67671bcdd92f2ae3cd364b2addefef70a3c2398

but bottom line is this shouldn't be here.

> +        bb.data.inheritFromOS(self.configuration.data)

and this line is the one that worries me since it pokes around the
environment. If we do this later, we're likely using a different
environment and I'm not happy this is going to give good results.

This actually ties into the terminal bbclass issues Chris has been
proposing recently. I think we need to store off the original
environment into a datastore and then leave it somewhere fixed we can
recall it later for the likes of this function and the terminal class
code.

> +        try:
> +            self.parseConfigurationFiles(self.configuration.prefile,
> +                                         self.configuration.postfile)
> +        except SyntaxError:
> +            sys.exit(1)
> +        except Exception:
> +            logger.exception("Error parsing configuration files")
> +            sys.exit(1)
> +
> +        if not self.configuration.cmd:
> +            self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build"
> +
>      def parseConfiguration(self):
>  
> 
> @@ -508,7 +515,7 @@ class BBCooker:
>          bb.data.expandKeys(localdata)
>  
>          # Handle PREFERRED_PROVIDERS
> -        for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
> +        for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, True) or "").split():
>              try:
>                  (providee, provider) = p.split(':')
>              except:
> @@ -999,8 +1006,8 @@ class BBCooker:
>  
>          self.server_registration_cb(buildTargetsIdle, rq)
>  
> -    def updateCache(self):
> -        if self.state == state.running:
> +    def updateCache(self, force=False):
> +        if self.state == state.running and not force:
>              return
>  
>          if self.state in (state.shutdown, state.stop):
> @@ -1010,6 +1017,8 @@ class BBCooker:
>          if self.state != state.parsing:
>              self.parseConfiguration ()
>  
> +            if self.status:
> +                del self.status
>              self.status = bb.cache.CacheData(self.caches_array)
>  
>              ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
> @@ -1083,7 +1092,7 @@ class BBCooker:
>  
>          collectlog.debug(1, "collecting .bb files")
>  
> -        files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
> +        files = (data.getVar( "BBFILES", self.configuration.data, True) or "").split()
>          data.setVar("BBFILES", " ".join(files), self.configuration.data)
>  
>          # Sort files by priority
> @@ -1140,7 +1149,8 @@ class BBCooker:
>              base = os.path.basename(f).replace('.bbappend', '.bb')
>              if not base in self.appendlist:
>                 self.appendlist[base] = []
> -            self.appendlist[base].append(f)
> +            if f not in self.appendlist[base]:
> +                self.appendlist[base].append(f)

append files only get appended once. Shouldn't that be a separate patch?

Otherwise the patch and the idea are ok, we just need to be careful to
get this right...

Cheers,

Richard

Re: [PATCH 3/6] command|cooker: Add reparseFiles command

[Joshua Lock]

On Fri, 2011-07-15 at 23:31 +0100, Richard Purdie wrote:
> On Fri, 2011-07-15 at 13:20 -0700, Joshua Lock wrote:
> > Add command reparseFiles to reparse bb files in a running cooker instance.
> > 
> > Fixes [YOCTO #1249]
> > 
> > Signed-off-by: Joshua Lock <josh@linux.intel.com>
> > ---
> >  lib/bb/command.py |    8 +++++++
> >  lib/bb/cooker.py  |   54 +++++++++++++++++++++++++++++++++-------------------
> >  2 files changed, 42 insertions(+), 20 deletions(-)
> > 
> 
> > diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
> > index 23f2c76..e4daaef 100644
> > --- a/lib/bb/cooker.py
> > +++ b/lib/bb/cooker.py
> > @@ -122,21 +122,8 @@ class BBCooker:
> >                  logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc))
> >                  sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name)
> >  
> > -        self.configuration.data = bb.data.init()
> > -
> > -        bb.data.inheritFromOS(self.configuration.data)
> > -
> > -        try:
> > -            self.parseConfigurationFiles(self.configuration.prefile,
> > -                                         self.configuration.postfile)
> > -        except SyntaxError:
> > -            sys.exit(1)
> > -        except Exception:
> > -            logger.exception("Error parsing configuration files")
> > -            sys.exit(1)
> > -
> > -        if not self.configuration.cmd:
> > -            self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build"
> > +        self.configuration.data = None
> > +        self.loadConfigurationData()
> >  
> >          bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True)
> >          if bbpkgs and len(self.configuration.pkgs_to_build) == 0:
> > @@ -163,6 +150,26 @@ class BBCooker:
> >  
> >          self.parser = None
> >  
> > +    def loadConfigurationData(self):
> > +        self.configuration.data = bb.data.init()
> > +
> > +        if not self.server_registration_cb:
> > +            bb.data.setVar("BB_WORKERCONTEXT", "1", self.configuration.data)
> 
> This looks like one of the last remaining differences between the
> bitbake in poky and the upstream. See:
> 
> http://git.openembedded.net/cgit.cgi/bitbake/commit/?h=master-poky&id=a67671bcdd92f2ae3cd364b2addefef70a3c2398
> 
> but bottom line is this shouldn't be here.

Whoops.

> 
> > +        bb.data.inheritFromOS(self.configuration.data)
> 
> and this line is the one that worries me since it pokes around the
> environment. If we do this later, we're likely using a different
> environment and I'm not happy this is going to give good results.
> 
> This actually ties into the terminal bbclass issues Chris has been
> proposing recently. I think we need to store off the original
> environment into a datastore and then leave it somewhere fixed we can
> recall it later for the likes of this function and the terminal class
> code.

Ah, that's a better idea. I'll take a stab at something to do that.

> 
> > +        try:
> > +            self.parseConfigurationFiles(self.configuration.prefile,
> > +                                         self.configuration.postfile)
> > +        except SyntaxError:
> > +            sys.exit(1)
> > +        except Exception:
> > +            logger.exception("Error parsing configuration files")
> > +            sys.exit(1)
> > +
> > +        if not self.configuration.cmd:
> > +            self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build"
> > +
> >      def parseConfiguration(self):
> >  
> > 
> > @@ -508,7 +515,7 @@ class BBCooker:
> >          bb.data.expandKeys(localdata)
> >  
> >          # Handle PREFERRED_PROVIDERS
> > -        for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
> > +        for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, True) or "").split():
> >              try:
> >                  (providee, provider) = p.split(':')
> >              except:
> > @@ -999,8 +1006,8 @@ class BBCooker:
> >  
> >          self.server_registration_cb(buildTargetsIdle, rq)
> >  
> > -    def updateCache(self):
> > -        if self.state == state.running:
> > +    def updateCache(self, force=False):
> > +        if self.state == state.running and not force:
> >              return
> >  
> >          if self.state in (state.shutdown, state.stop):
> > @@ -1010,6 +1017,8 @@ class BBCooker:
> >          if self.state != state.parsing:
> >              self.parseConfiguration ()
> >  
> > +            if self.status:
> > +                del self.status
> >              self.status = bb.cache.CacheData(self.caches_array)
> >  
> >              ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
> > @@ -1083,7 +1092,7 @@ class BBCooker:
> >  
> >          collectlog.debug(1, "collecting .bb files")
> >  
> > -        files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
> > +        files = (data.getVar( "BBFILES", self.configuration.data, True) or "").split()
> >          data.setVar("BBFILES", " ".join(files), self.configuration.data)
> >  
> >          # Sort files by priority
> > @@ -1140,7 +1149,8 @@ class BBCooker:
> >              base = os.path.basename(f).replace('.bbappend', '.bb')
> >              if not base in self.appendlist:
> >                 self.appendlist[base] = []
> > -            self.appendlist[base].append(f)
> > +            if f not in self.appendlist[base]:
> > +                self.appendlist[base].append(f)
> 
> append files only get appended once. Shouldn't that be a separate patch?

Quite possibly. I have a bad habit of spotting things and changing them
when I should be thinking about something else. I'll split this change
out next time. Must have missed it this try.

> 
> Otherwise the patch and the idea are ok, we just need to be careful to
> get this right...

Thanks for the detailed review. I'll work on a new revision to address
your concerns.

Regards,
Joshua
-- 
Joshua Lock
        Yocto Project "Johannes factotum"
        Intel Open Source Technology Centre

[PATCH 4/6] ui/crumbs/hobeventhandler: reparse files before running other commands

[Joshua Lock]

Integrate reparseFiles into the run_next_command() method rather than calling
reparseFiles on the server and immediately calling other methods.

Signed-off-by: Joshua Lock <josh@linux.intel.com>
---
 lib/bb/ui/crumbs/hobeventhandler.py |    8 +++++---
 1 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/bb/ui/crumbs/hobeventhandler.py b/lib/bb/ui/crumbs/hobeventhandler.py
index 50f3419..8a17e37 100644
--- a/lib/bb/ui/crumbs/hobeventhandler.py
+++ b/lib/bb/ui/crumbs/hobeventhandler.py
@@ -61,7 +61,7 @@ class HobHandler(gobject.GObject):
                                    gobject.TYPE_STRING)),
     }
 
-    (CFG_PATH_LOCAL, CFG_PATH_HOB, CFG_PATH_LAYERS, CFG_FILES_DISTRO, CFG_FILES_MACH, CFG_FILES_SDK, FILES_MATCH_CLASS, GENERATE_TGTS) = range(8)
+    (CFG_PATH_LOCAL, CFG_PATH_HOB, CFG_PATH_LAYERS, CFG_FILES_DISTRO, CFG_FILES_MACH, CFG_FILES_SDK, FILES_MATCH_CLASS, GENERATE_TGTS, REPARSE_FILES) = range(9)
 
     def __init__(self, taskmodel, server):
         gobject.GObject.__init__(self)
@@ -108,6 +108,9 @@ class HobHandler(gobject.GObject):
                 self.emit("data-generated")
                 self.generating = False
             self.current_command = None
+        elif self.current_command == self.REPARSE_FILES:
+            self.current_command = self.CFG_PATH_LAYERS
+            self.server.runCommand(["reparseFiles"])
         else:
             # No command?
             pass
@@ -196,8 +199,7 @@ class HobHandler(gobject.GObject):
         img = self.model.selected_image
         selected_packages, _ = self.model.get_selected_packages()
         self.emit("reload-triggered", img, " ".join(selected_packages))
-        self.server.runCommand(["reparseFiles"])
-        self.current_command = self.CFG_PATH_LAYERS
+        self.current_command = self.REPARSE_FILES
         self.run_next_command()
 
     def set_bbthreads(self, threads):
-- 
1.7.6

[PATCH 5/6] ui/crumbs/tasklistmodel: fix saving recipes

[Joshua Lock]

After switching to dynamically finding the relative path for the recipe
file it's no longer to append .bb when inserting the require line into the
saved recipe.

Fixes [YOCTO #1247]

Signed-off-by: Joshua Lock <josh@linux.intel.com>
---
 lib/bb/ui/crumbs/tasklistmodel.py |    3 +--
 1 files changed, 1 insertions(+), 2 deletions(-)

diff --git a/lib/bb/ui/crumbs/tasklistmodel.py b/lib/bb/ui/crumbs/tasklistmodel.py
index 633931d..e28dbe7 100644
--- a/lib/bb/ui/crumbs/tasklistmodel.py
+++ b/lib/bb/ui/crumbs/tasklistmodel.py
@@ -53,11 +53,10 @@ class BuildRep(gobject.GObject):
         self.userpkgs = packages
 
     def writeRecipe(self, writepath, model):
-        # FIXME: Need a better way to determine meta_path...
         template = """
 # Recipe generated by the HOB
 
-require %s.bb
+require %s
 
 IMAGE_INSTALL += "%s"
 """
-- 
1.7.6

[PATCH 6/6] ui/hob: switch from buildFile to buildTargets for custom image builds

[Joshua Lock]

We need to use the buildTargets command to ensure dependencies, such as
native tools to build the rootfs, are correctly included. This patch
achieves this by modifying BBPATH and BBFILES to include matches for the
location of the generated recipe file and reparsing the metadata before
calling buildTargets.

Fixes [YOCTO #1228]

Signed-off-by: Joshua Lock <josh@linux.intel.com>
---
 lib/bb/ui/crumbs/configurator.py    |   18 +++++++
 lib/bb/ui/crumbs/hobeventhandler.py |   84 +++++++++++++++++++++--------------
 lib/bb/ui/hob.py                    |   24 ++++------
 3 files changed, 78 insertions(+), 48 deletions(-)

diff --git a/lib/bb/ui/crumbs/configurator.py b/lib/bb/ui/crumbs/configurator.py
index b694143..b3ea76c 100644
--- a/lib/bb/ui/crumbs/configurator.py
+++ b/lib/bb/ui/crumbs/configurator.py
@@ -240,6 +240,24 @@ class Configurator(gobject.GObject):
         del self.orig_config
         self.orig_config = copy.deepcopy(self.config)
 
+    def insertTempBBPath(self, bbpath, bbfiles):
+        # Create a backup of the local.conf
+        bkup = "%s~" % self.local
+        os.rename(self.local, bkup)
+
+        # read the original conf into a list
+        with open(bkup, 'r') as config:
+            config_lines = config.readlines()
+
+        if bbpath:
+            config_lines.append("BBPATH := \"${BBPATH}:%s\"\n" % bbpath)
+        if bbfiles:
+            config_lines.append("BBFILES := \"${BBFILES} %s\"\n" % bbfiles)
+
+        # Write the updated lines list object to the local.conf
+        with open(self.local, "w") as n:
+            n.write("".join(config_lines))
+
     def writeLayerConf(self):
         # If we've not added/removed new layers don't write
         if not self._isLayerConfDirty():
diff --git a/lib/bb/ui/crumbs/hobeventhandler.py b/lib/bb/ui/crumbs/hobeventhandler.py
index 8a17e37..f341826 100644
--- a/lib/bb/ui/crumbs/hobeventhandler.py
+++ b/lib/bb/ui/crumbs/hobeventhandler.py
@@ -52,16 +52,13 @@ class HobHandler(gobject.GObject):
          "error"               : (gobject.SIGNAL_RUN_LAST,
                                   gobject.TYPE_NONE,
                                   (gobject.TYPE_STRING,)),
-         "build-complete"      : (gobject.SIGNAL_RUN_LAST,
-                                  gobject.TYPE_NONE,
-                                  ()),
          "reload-triggered"    : (gobject.SIGNAL_RUN_LAST,
                                   gobject.TYPE_NONE,
                                   (gobject.TYPE_STRING,
                                    gobject.TYPE_STRING)),
     }
 
-    (CFG_PATH_LOCAL, CFG_PATH_HOB, CFG_PATH_LAYERS, CFG_FILES_DISTRO, CFG_FILES_MACH, CFG_FILES_SDK, FILES_MATCH_CLASS, GENERATE_TGTS, REPARSE_FILES) = range(9)
+    (CFG_PATH_LOCAL, CFG_PATH_HOB, CFG_PATH_LAYERS, CFG_FILES_DISTRO, CFG_FILES_MACH, CFG_FILES_SDK, FILES_MATCH_CLASS, GENERATE_TGTS, REPARSE_FILES, BUILD_IMAGE) = range(10)
 
     def __init__(self, taskmodel, server):
         gobject.GObject.__init__(self)
@@ -109,8 +106,21 @@ class HobHandler(gobject.GObject):
                 self.generating = False
             self.current_command = None
         elif self.current_command == self.REPARSE_FILES:
-            self.current_command = self.CFG_PATH_LAYERS
+            if self.build_queue:
+                self.current_command = self.BUILD_IMAGE
+            else:
+                self.current_command = self.CFG_PATH_LAYERS
             self.server.runCommand(["reparseFiles"])
+        elif self.current_command == self.BUILD_IMAGE:
+            self.building = "image"
+            if self.generating:
+                self.emit("data-generated")
+                self.generating = False
+            bbpath = self.server.runCommand(["getVariable", "BBPATH"])
+            bbfiles = self.server.runCommand(["getVariable", "BBFILES"])
+            self.server.runCommand(["buildTargets", self.build_queue, "build"])
+            self.build_queue = []
+            self.current_command = None
         else:
             # No command?
             pass
@@ -209,27 +219,49 @@ class HobHandler(gobject.GObject):
         pmake = "-j %s" % threads
         self.server.runCommand(["setVariable", "BB_NUMBER_THREADS", pmake])
 
-    def run_build(self, tgts):
-        self.building = "image"
+    def build_image(self, image, image_path, configurator):
         targets = []
-        targets.append(tgts)
+        targets.append(image)
         if self.build_toolchain and self.build_toolchain_headers:
-            targets = ["meta-toolchain-sdk"] + targets
+            targets.append("meta-toolchain-sdk")
         elif self.build_toolchain:
-            targets = ["meta-toolchain"] + targets
-        self.server.runCommand(["buildTargets", targets, "build"])
+            targets.append("meta-toolchain")
+        self.build_queue = targets
+
+        bbpath_ok = False
+        bbpath = self.server.runCommand(["getVariable", "BBPATH"])
+        if image_path in bbpath.split(":"):
+            bbpath_ok = True
+
+        bbfiles_ok = False
+        bbfiles = self.server.runCommand(["getVariable", "BBFILES"]).split(" ")
+        for files in bbfiles:
+            import re
+            if re.match(files, re.escape("%s/*.bb" % image_path)):
+                bbfile_ok = True
+
+        if not bbpath_ok:
+            nbbp = image_path
+        else:
+            nbbp = None
+
+        if not bbfiles_ok:
+            nbbf = "%s/*.bb" % image_path
+        else:
+            nbbf = None
+
+        if bbfiles_ok and bbpath_ok:
+            self.building = "image"
+            self.current_command = self.BUILD_IMAGE
+        else:
+            configurator.insertTempBBPath(nbbp, nbbf)
+            self.current_command = self.REPARSE_FILES
+        self.run_next_command()
 
     def build_packages(self, pkgs):
         self.building = "packages"
-        if 'meta-toolchain' in self.build_queue:
-            self.build_queue.remove('meta-toolchain')
-            pkgs.extend('meta-toolchain')
         self.server.runCommand(["buildTargets", pkgs, "build"])
 
-    def build_file(self, image):
-        self.building = "image"
-        self.server.runCommand(["buildFile", image, "build"])
-
     def cancel_build(self, force=False):
         if force:
             # Force the cooker to stop as quickly as possible
@@ -255,22 +287,6 @@ class HobHandler(gobject.GObject):
         if self.build_toolchain_headers != enabled:
             self.build_toolchain_headers = enabled
 
-    def queue_image_recipe_path(self, path):
-        self.build_queue.append(path)
-
-    def build_complete_cb(self, running_build):
-        if len(self.build_queue) > 0:
-            next = self.build_queue.pop(0)
-            if next.endswith('.bb'):
-                self.build_file(next)
-                self.building = 'image'
-                self.build_file(next)
-            else:
-                self.build_packages(next.split(" "))
-        else:
-            self.building = None
-            self.emit("build-complete")
-
     def set_image_output_type(self, output_type):
         self.server.runCommand(["setVariable", "IMAGE_FSTYPES", output_type])
 
diff --git a/lib/bb/ui/hob.py b/lib/bb/ui/hob.py
index 888f42e..159f985 100644
--- a/lib/bb/ui/hob.py
+++ b/lib/bb/ui/hob.py
@@ -64,10 +64,8 @@ class MainWindow (gtk.Window):
 
         self.build = RunningBuild()
         self.build.connect("build-failed", self.running_build_failed_cb)
-        self.build.connect("build-complete", self.handler.build_complete_cb)
         self.build.connect("build-started", self.build_started_cb)
-
-        self.handler.connect("build-complete", self.build_complete_cb)
+        self.build.connect("build-complete", self.build_complete_cb)
 
         vbox = gtk.VBox(False, 0)
         vbox.set_border_width(0)
@@ -371,16 +369,15 @@ class MainWindow (gtk.Window):
             dialog.destroy()
             if response == gtk.RESPONSE_CANCEL:
                 return
-        else:
-            # TODO: show a confirmation dialog ?
-            if not self.save_path:
-                import tempfile, datetime
-                image_name = "hob-%s-variant-%s.bb" % (rep.base_image, datetime.date.today().isoformat())
-                image_dir = os.path.join(tempfile.gettempdir(), 'hob-images')
-                bb.utils.mkdirhier(image_dir)
-                recipepath =  os.path.join(image_dir, image_name)
             else:
-                recipepath = self.save_path
+                self.handler.build_packages(rep.allpkgs.split(" "))
+        else:
+            import tempfile, datetime
+            image_name = "hob-%s-variant-%s" % (rep.base_image, datetime.date.today().isoformat())
+            image_file = "%s.bb" % (image_name)
+            image_dir = os.path.join(tempfile.gettempdir(), 'hob-images')
+            bb.utils.mkdirhier(image_dir)
+            recipepath =  os.path.join(image_dir, image_file)
 
             rep.writeRecipe(recipepath, self.model)
             # In the case where we saved the file for the purpose of building
@@ -389,9 +386,8 @@ class MainWindow (gtk.Window):
             if not self.save_path:
                 self.files_to_clean.append(recipepath)
 
-            self.handler.queue_image_recipe_path(recipepath)
+            self.handler.build_image(image_name, image_dir, self.configurator)
 
-        self.handler.build_packages(rep.allpkgs.split(" "))
         self.nb.set_current_page(1)
 
     def back_button_clicked_cb(self, button):
-- 
1.7.6