[PATCH 0/9] devtool fixes

[PATCH 0/9] devtool fixes

[Paul Eggleton]

Some fixes and tidy-ups for devtool. This squashes together some of
Markus's code cleanup patches (minus the line wrapping ones), brings
in one of the patches from his local files series, and adds some new
fixes from me.


The following changes since commit 90861b8908d254154f4d1d613471070df8013da8:

  base: Avoid find race (2015-05-14 15:22:42 +0100)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib paule/devtool-fixes3
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/devtool-fixes3

Markus Lehtonen (5):
  devtool: extract: remove patches when S=WORKDIR
  devtool: remove unused imports / re-imports
  devtool: rename unused variables
  devtool: add missing docstrings
  devtool: deploy plugin: fix bad indentation

Paul Eggleton (4):
  devtool: fix build env command execution error handling
  lib/oe/recipeutils: add a parse_recipe_simple() function
  devtool: deploy-target: use tinfoil instead of bitbake -e
  devtool: if workspace layer exists, still ensure it's in bblayers.conf

 meta/lib/oe/recipeutils.py      | 29 +++++++++++++++--
 scripts/devtool                 | 36 ++++++++++++---------
 scripts/lib/devtool/__init__.py | 15 ++++++---
 scripts/lib/devtool/deploy.py   | 25 ++++++++++-----
 scripts/lib/devtool/standard.py | 70 ++++++++++++++++++++++++++++++++++++-----
 5 files changed, 138 insertions(+), 37 deletions(-)

-- 
2.1.0

[PATCH 1/9] devtool: extract: remove patches when S=WORKDIR

[Paul Eggleton]

From: Markus Lehtonen <markus.lehtonen@linux.intel.com>

Before this change, all files from the recipe (SRC_URI), including
patches, were added to to srctree repository when S==WORKDIR. The patch
files are useless as they are automatically applied on top of the
srctree by devtool.

This change causes devtool extract to not commit these unnecessary (and
possibly confusing) patch file(s) into srctree repository.

[YOCTO #7602]

Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
 scripts/lib/devtool/standard.py | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 81a44d4..3bc84c7 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -177,6 +177,16 @@ def _parse_recipe(config, tinfoil, pn, appends):
     return oe.recipeutils.parse_recipe(recipefile, append_files,
                                        tinfoil.config_data)
 
+
+def _ls_tree(directory):
+    """Recursive listing of files in a directory"""
+    ret = []
+    for root, dirs, files in os.walk(directory):
+        ret.extend([os.path.relpath(os.path.join(root, fname), directory) for
+                    fname in files])
+    return ret
+
+
 def extract(args, config, basepath, workspace):
     import bb
 
@@ -196,6 +206,7 @@ def extract(args, config, basepath, workspace):
 
 def _extract_source(srctree, keep_temp, devbranch, d):
     import bb.event
+    import oe.recipeutils
 
     def eventfilter(name, handler, event, d):
         if name == 'base_eventhandler':
@@ -264,7 +275,21 @@ def _extract_source(srctree, keep_temp, devbranch, d):
         logger.info('Unpacking...')
         exec_task_func('do_unpack', False)
         srcsubdir = crd.getVar('S', True)
-        if srcsubdir != workdir and os.path.dirname(srcsubdir) != workdir:
+        if srcsubdir == workdir:
+            # Find non-patch sources that were "unpacked" to srctree directory
+            recipe_patches = [os.path.basename(patch) for patch in
+                              oe.recipeutils.get_recipe_patches(crd)]
+            src_files = [fname for fname in _ls_tree(workdir) if
+                         os.path.basename(fname) not in recipe_patches]
+            # Force separate S so that patch files can be left out from srctree
+            srcsubdir = tempfile.mkdtemp(dir=workdir)
+            crd.setVar('S', srcsubdir)
+            # Move source files to S
+            for path in src_files:
+                tgt_dir = os.path.join(srcsubdir, os.path.dirname(path))
+                bb.utils.mkdirhier(tgt_dir)
+                shutil.move(os.path.join(workdir, path), tgt_dir)
+        elif os.path.dirname(srcsubdir) != workdir:
             # Handle if S is set to a subdirectory of the source
             srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
 
-- 
2.1.0

[PATCH 2/9] devtool: remove unused imports / re-imports

[Paul Eggleton]

From: Markus Lehtonen <markus.lehtonen@linux.intel.com>

Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
 scripts/lib/devtool/__init__.py | 1 -
 scripts/lib/devtool/deploy.py   | 1 -
 scripts/lib/devtool/standard.py | 1 -
 3 files changed, 3 deletions(-)

diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py
index 78ae0aa..f19d032 100644
--- a/scripts/lib/devtool/__init__.py
+++ b/scripts/lib/devtool/__init__.py
@@ -75,7 +75,6 @@ def setup_tinfoil():
         sys.exit(1)
 
     import bb.tinfoil
-    import logging
     tinfoil = bb.tinfoil.Tinfoil()
     tinfoil.prepare(False)
     tinfoil.logger.setLevel(logging.WARNING)
diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py
index f016b23..e46ee05 100644
--- a/scripts/lib/devtool/deploy.py
+++ b/scripts/lib/devtool/deploy.py
@@ -28,7 +28,6 @@ def plugin_init(pluginlist):
 
 def deploy(args, config, basepath, workspace):
     import re
-    from devtool import exec_build_env_command
 
     if not args.recipename in workspace:
         logger.error("no recipe named %s in your workspace" % args.recipename)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 3bc84c7..3e7b54e 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -19,7 +19,6 @@ import os
 import sys
 import re
 import shutil
-import glob
 import tempfile
 import logging
 import argparse
-- 
2.1.0

[PATCH 3/9] devtool: rename unused variables

[Paul Eggleton]

From: Markus Lehtonen <markus.lehtonen@linux.intel.com>

Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
 scripts/lib/devtool/deploy.py   | 4 ++--
 scripts/lib/devtool/standard.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py
index e46ee05..4f968c6 100644
--- a/scripts/lib/devtool/deploy.py
+++ b/scripts/lib/devtool/deploy.py
@@ -42,7 +42,7 @@ def deploy(args, config, basepath, workspace):
     deploy_dir = os.path.join(basepath, 'target_deploy', args.target)
     deploy_file = os.path.join(deploy_dir, args.recipename + '.list')
 
-    stdout, stderr = exec_build_env_command(config.init_path, basepath, 'bitbake -e %s' % args.recipename, shell=True)
+    stdout, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -e %s' % args.recipename, shell=True)
     recipe_outdir = re.search(r'^D="(.*)"', stdout, re.MULTILINE).group(1)
     if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
         logger.error('No files to deploy - have you built the %s recipe? If so, the install step has not installed any files.' % args.recipename)
@@ -50,7 +50,7 @@ def deploy(args, config, basepath, workspace):
 
     if args.dry_run:
         print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
-        for root, dirs, files in os.walk(recipe_outdir):
+        for root, _, files in os.walk(recipe_outdir):
             for fn in files:
                 print('  %s' % os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn))
         return 0
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 3e7b54e..a9dd3b2 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -94,7 +94,7 @@ def add(args, config, basepath, workspace):
         source = srctree
     if args.version:
         extracmdopts += ' -V %s' % args.version
-    stdout, stderr = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, recipefile, source, extracmdopts))
+    stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, recipefile, source, extracmdopts))
     logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
 
     _add_md5(config, args.recipename, recipefile)
-- 
2.1.0

[PATCH 4/9] devtool: add missing docstrings

[Paul Eggleton]

From: Markus Lehtonen <markus.lehtonen@linux.intel.com>

Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
 scripts/lib/devtool/__init__.py |  5 ++++-
 scripts/lib/devtool/deploy.py   |  6 +++++-
 scripts/lib/devtool/standard.py | 20 +++++++++++++++++++-
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py
index f19d032..8866512 100644
--- a/scripts/lib/devtool/__init__.py
+++ b/scripts/lib/devtool/__init__.py
@@ -16,7 +16,7 @@
 # You should have received a copy of the GNU General Public License along
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
+"""Devtool plugins module"""
 
 import os
 import sys
@@ -26,6 +26,7 @@ import logging
 logger = logging.getLogger('devtool')
 
 def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
+    """Run a program in bitbake build context"""
     import bb
     if not 'cwd' in options:
         options["cwd"] = builddir
@@ -49,6 +50,7 @@ def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
         return bb.process.run('%s%s' % (init_prefix, cmd), **options)
 
 def exec_watch(cmd, **options):
+    """Run program with stdout shown on sys.stdout"""
     if isinstance(cmd, basestring) and not "shell" in options:
         options["shell"] = True
 
@@ -68,6 +70,7 @@ def exec_watch(cmd, **options):
     return buf
 
 def setup_tinfoil():
+    """Initialize tinfoil api from bitbake"""
     import scriptpath
     bitbakepath = scriptpath.add_bitbake_lib_path()
     if not bitbakepath:
diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py
index 4f968c6..3c7abfa 100644
--- a/scripts/lib/devtool/deploy.py
+++ b/scripts/lib/devtool/deploy.py
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License along
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+"""Devtool plugin containing the deploy subcommands"""
 
 import os
 import subprocess
@@ -23,10 +24,12 @@ from devtool import exec_build_env_command
 logger = logging.getLogger('devtool')
 
 def plugin_init(pluginlist):
+    """Plugin initialization"""
     pass
 
 
 def deploy(args, config, basepath, workspace):
+    """Entry point for the devtool 'deploy' subcommand"""
     import re
 
     if not args.recipename in workspace:
@@ -87,7 +90,7 @@ def deploy(args, config, basepath, workspace):
     return 0
 
 def undeploy(args, config, basepath, workspace):
-
+    """Entry point for the devtool 'undeploy' subcommand"""
     deploy_file = os.path.join(basepath, 'target_deploy', args.target, args.recipename + '.list')
     if not os.path.exists(deploy_file):
          logger.error('%s has not been deployed' % args.recipename)
@@ -122,6 +125,7 @@ def undeploy(args, config, basepath, workspace):
 
 
 def register_commands(subparsers, context):
+    """Register devtool subcommands from the deploy plugin"""
     parser_deploy = subparsers.add_parser('deploy-target', help='Deploy recipe output files to live target machine')
     parser_deploy.add_argument('recipename', help='Recipe to deploy')
     parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index a9dd3b2..2f8b194 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License along
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+"""Devtool standard plugins"""
 
 import os
 import sys
@@ -28,10 +29,12 @@ from devtool import exec_build_env_command, setup_tinfoil
 logger = logging.getLogger('devtool')
 
 def plugin_init(pluginlist):
+    """Plugin initialization"""
     pass
 
 
 def add(args, config, basepath, workspace):
+    """Entry point for the devtool 'add' subcommand"""
     import bb
     import oe.recipeutils
 
@@ -119,6 +122,7 @@ def add(args, config, basepath, workspace):
 
 
 def _check_compatible_recipe(pn, d):
+    """Check if the recipe is supported by devtool"""
     if pn == 'perf':
         logger.error("The perf recipe does not actually check out source and thus cannot be supported by this tool")
         return False
@@ -151,6 +155,7 @@ def _check_compatible_recipe(pn, d):
 
 
 def _get_recipe_file(cooker, pn):
+    """Find recipe file corresponding a package name"""
     import oe.recipeutils
     recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
     if not recipefile:
@@ -187,6 +192,7 @@ def _ls_tree(directory):
 
 
 def extract(args, config, basepath, workspace):
+    """Entry point for the devtool 'extract' subcommand"""
     import bb
 
     tinfoil = setup_tinfoil()
@@ -204,10 +210,12 @@ def extract(args, config, basepath, workspace):
 
 
 def _extract_source(srctree, keep_temp, devbranch, d):
+    """Extract sources of a recipe"""
     import bb.event
     import oe.recipeutils
 
     def eventfilter(name, handler, event, d):
+        """Bitbake event filter for devtool extract operation"""
         if name == 'base_eventhandler':
             return True
         else:
@@ -257,6 +265,7 @@ def _extract_source(srctree, keep_temp, devbranch, d):
         # are to handle e.g. linux-yocto's extra tasks
         executed = []
         def exec_task_func(func, report):
+            """Run specific bitbake task for a recipe"""
             if not func in executed:
                 deps = crd.getVarFlag(func, 'deps')
                 if deps:
@@ -343,12 +352,15 @@ def _extract_source(srctree, keep_temp, devbranch, d):
     return initial_rev
 
 def _add_md5(config, recipename, filename):
+    """Record checksum of a recipe to the md5-file of the workspace"""
     import bb.utils
     md5 = bb.utils.md5_file(filename)
     with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a') as f:
         f.write('%s|%s|%s\n' % (recipename, os.path.relpath(filename, config.workspace_path), md5))
 
 def _check_preserve(config, recipename):
+    """Check if a recipe was manually changed and needs to be saved in 'attic'
+       directory"""
     import bb.utils
     origfile = os.path.join(config.workspace_path, '.devtool_md5')
     newfile = os.path.join(config.workspace_path, '.devtool_md5_new')
@@ -382,6 +394,7 @@ def _check_preserve(config, recipename):
 
 
 def modify(args, config, basepath, workspace):
+    """Entry point for the devtool 'modify' subcommand"""
     import bb
     import oe.recipeutils
 
@@ -481,6 +494,7 @@ def modify(args, config, basepath, workspace):
 
 
 def update_recipe(args, config, basepath, workspace):
+    """Entry point for the devtool 'update-recipe' subcommand"""
     if not args.recipename in workspace:
         logger.error("no recipe named %s in your workspace" % args.recipename)
         return -1
@@ -511,7 +525,7 @@ def update_recipe(args, config, basepath, workspace):
         mode = args.mode
 
     def remove_patches(srcuri, patchlist):
-        # Remove any patches that we don't need
+        """Remove patches"""
         updated = False
         for patch in patchlist:
             patchfile = os.path.basename(patch)
@@ -663,6 +677,7 @@ def update_recipe(args, config, basepath, workspace):
 
 
 def status(args, config, basepath, workspace):
+    """Entry point for the devtool 'status' subcommand"""
     if workspace:
         for recipe, value in workspace.iteritems():
             print("%s: %s" % (recipe, value))
@@ -672,6 +687,7 @@ def status(args, config, basepath, workspace):
 
 
 def reset(args, config, basepath, workspace):
+    """Entry point for the devtool 'reset' subcommand"""
     import bb.utils
     if args.recipename:
         if args.all:
@@ -713,6 +729,7 @@ def reset(args, config, basepath, workspace):
 
 
 def build(args, config, basepath, workspace):
+    """Entry point for the devtool 'build' subcommand"""
     import bb
     if not args.recipename in workspace:
         logger.error("no recipe named %s in your workspace" % args.recipename)
@@ -724,6 +741,7 @@ def build(args, config, basepath, workspace):
 
 
 def register_commands(subparsers, context):
+    """Register devtool subcommands from this plugin"""
     parser_add = subparsers.add_parser('add', help='Add a new recipe',
                                        description='Adds a new recipe')
     parser_add.add_argument('recipename', help='Name for new recipe to add')
-- 
2.1.0

[PATCH 5/9] devtool: deploy plugin: fix bad indentation

[Paul Eggleton]

From: Markus Lehtonen <markus.lehtonen@linux.intel.com>

Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
 scripts/lib/devtool/deploy.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py
index 3c7abfa..078c74b 100644
--- a/scripts/lib/devtool/deploy.py
+++ b/scripts/lib/devtool/deploy.py
@@ -93,8 +93,8 @@ def undeploy(args, config, basepath, workspace):
     """Entry point for the devtool 'undeploy' subcommand"""
     deploy_file = os.path.join(basepath, 'target_deploy', args.target, args.recipename + '.list')
     if not os.path.exists(deploy_file):
-         logger.error('%s has not been deployed' % args.recipename)
-         return -1
+        logger.error('%s has not been deployed' % args.recipename)
+        return -1
 
     if args.dry_run:
         print('Previously deployed files to be un-deployed for %s on target %s:' % (args.recipename, args.target))
-- 
2.1.0

[PATCH 6/9] devtool: fix build env command execution error handling

[Paul Eggleton]

If we execute an external command, we ought to prepare for the
possibility that it can fail and handle the failure appropriately. We
can especially expect this to happen when running bitbake in this
scenario. Ensure we return the appropriate exit code to the calling
process.

Fixes [YOCTO #7757].

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/devtool/__init__.py |  9 +++++++--
 scripts/lib/devtool/standard.py | 22 +++++++++++++++++-----
 2 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py
index 8866512..5a06c78 100644
--- a/scripts/lib/devtool/__init__.py
+++ b/scripts/lib/devtool/__init__.py
@@ -44,13 +44,14 @@ def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
     if watch:
         if sys.stdout.isatty():
             # Fool bitbake into thinking it's outputting to a terminal (because it is, indirectly)
-            cmd = 'script -q -c "%s" /dev/null' % cmd
+            cmd = 'script -e -q -c "%s" /dev/null' % cmd
         return exec_watch('%s%s' % (init_prefix, cmd), **options)
     else:
         return bb.process.run('%s%s' % (init_prefix, cmd), **options)
 
 def exec_watch(cmd, **options):
     """Run program with stdout shown on sys.stdout"""
+    import bb
     if isinstance(cmd, basestring) and not "shell" in options:
         options["shell"] = True
 
@@ -67,7 +68,11 @@ def exec_watch(cmd, **options):
             buf += out
         elif out == '' and process.poll() != None:
             break
-    return buf
+
+    if process.returncode != 0:
+        raise bb.process.ExecutionError(cmd, process.returncode, buf, None)
+
+    return buf, None
 
 def setup_tinfoil():
     """Initialize tinfoil api from bitbake"""
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 2f8b194..61c0df9 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -97,8 +97,12 @@ def add(args, config, basepath, workspace):
         source = srctree
     if args.version:
         extracmdopts += ' -V %s' % args.version
-    stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, recipefile, source, extracmdopts))
-    logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
+    try:
+        stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, recipefile, source, extracmdopts))
+        logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
+    except bb.process.ExecutionError as e:
+        logger.error('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
+        return 1
 
     _add_md5(config, args.recipename, recipefile)
 
@@ -688,7 +692,7 @@ def status(args, config, basepath, workspace):
 
 def reset(args, config, basepath, workspace):
     """Entry point for the devtool 'reset' subcommand"""
-    import bb.utils
+    import bb
     if args.recipename:
         if args.all:
             logger.error("Recipe cannot be specified if -a/--all is used")
@@ -708,7 +712,11 @@ def reset(args, config, basepath, workspace):
     for pn in recipes:
         if not args.no_clean:
             logger.info('Cleaning sysroot for recipe %s...' % pn)
-            exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % pn)
+            try:
+                exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % pn)
+            except bb.process.ExecutionError as e:
+                logger.error('Command \'%s\' failed, output:\n%s\nIf you wish, you may specify -n/--no-clean to skip running this command when resetting' % (e.command, e.stdout))
+                return 1
 
         _check_preserve(config, pn)
 
@@ -735,7 +743,11 @@ def build(args, config, basepath, workspace):
         logger.error("no recipe named %s in your workspace" % args.recipename)
         return -1
     build_task = config.get('Build', 'build_task', 'populate_sysroot')
-    exec_build_env_command(config.init_path, basepath, 'bitbake -c %s %s' % (build_task, args.recipename), watch=True)
+    try:
+        exec_build_env_command(config.init_path, basepath, 'bitbake -c %s %s' % (build_task, args.recipename), watch=True)
+    except bb.process.ExecutionError as e:
+        # We've already seen the output since watch=True, so just ensure we return something to the user
+        return e.exitcode
 
     return 0
 
-- 
2.1.0

[PATCH 7/9] lib/oe/recipeutils: add a parse_recipe_simple() function

[Paul Eggleton]

Add a function that simply parses a recipe by name and optionally the
bbappends that apply to it. (Note that if you're using tinfoil you need
to have initialised it with config_only=False so that it can map the
recipe name to a recipe file.)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 meta/lib/oe/recipeutils.py | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)

diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index 19d97b6..0689fb0 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -44,13 +44,36 @@ def get_unavailable_reasons(cooker, pn):
     return taskdata.get_reasons(pn)
 
 
-def parse_recipe(fn, appends, d):
-    """Parse an individual recipe"""
+def parse_recipe(fn, appendfiles, d):
+    """
+    Parse an individual recipe file, optionally with a list of
+    bbappend files.
+    """
     import bb.cache
-    envdata = bb.cache.Cache.loadDataFull(fn, appends, d)
+    envdata = bb.cache.Cache.loadDataFull(fn, appendfiles, d)
     return envdata
 
 
+def parse_recipe_simple(cooker, pn, d, appends=True):
+    """
+    Parse a recipe and optionally all bbappends that apply to it
+    in the current configuration.
+    """
+    import bb.providers
+
+    recipefile = pn_to_recipe(cooker, pn)
+    if not recipefile:
+        skipreasons = get_unavailable_reasons(cooker, pn)
+        # We may as well re-use bb.providers.NoProvider here
+        if skipreasons:
+            raise bb.providers.NoProvider(skipreasons)
+        else:
+            raise bb.providers.NoProvider('Unable to find any recipe file matching %s' % pn)
+    if appends:
+        appendfiles = cooker.collection.get_file_appends(recipefile)
+    return parse_recipe(recipefile, appendfiles, d)
+
+
 def get_var_files(fn, varlist, d):
     """Find the file in which each of a list of variables is set.
     Note: requires variable history to be enabled when parsing.
-- 
2.1.0

[PATCH 8/9] devtool: deploy-target: use tinfoil instead of bitbake -e

[Paul Eggleton]

Using tinfoil here is quicker and tidier than shelling out to
bitbake -e and interpreting its output.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/devtool/deploy.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py
index 078c74b..92a3cb4 100644
--- a/scripts/lib/devtool/deploy.py
+++ b/scripts/lib/devtool/deploy.py
@@ -19,7 +19,7 @@
 import os
 import subprocess
 import logging
-from devtool import exec_build_env_command
+from devtool import exec_build_env_command, setup_tinfoil
 
 logger = logging.getLogger('devtool')
 
@@ -31,6 +31,7 @@ def plugin_init(pluginlist):
 def deploy(args, config, basepath, workspace):
     """Entry point for the devtool 'deploy' subcommand"""
     import re
+    import oe.recipeutils
 
     if not args.recipename in workspace:
         logger.error("no recipe named %s in your workspace" % args.recipename)
@@ -45,8 +46,13 @@ def deploy(args, config, basepath, workspace):
     deploy_dir = os.path.join(basepath, 'target_deploy', args.target)
     deploy_file = os.path.join(deploy_dir, args.recipename + '.list')
 
-    stdout, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -e %s' % args.recipename, shell=True)
-    recipe_outdir = re.search(r'^D="(.*)"', stdout, re.MULTILINE).group(1)
+    tinfoil = setup_tinfoil()
+    try:
+        rd = oe.recipeutils.parse_recipe_simple(tinfoil.cooker, args.recipename, tinfoil.config_data)
+    except Exception as e:
+        logger.error('Exception parsing recipe %s: %s' % (args.recipename, e))
+        return 2
+    recipe_outdir = rd.getVar('D', True)
     if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
         logger.error('No files to deploy - have you built the %s recipe? If so, the install step has not installed any files.' % args.recipename)
         return -1
-- 
2.1.0

[PATCH 9/9] devtool: if workspace layer exists, still ensure it's in bblayers.conf

[Paul Eggleton]

When we run devtool, if the workspace layer already exists but isn't in
bblayers.conf (perhaps because it was previously created but
subsequently removed from bblayers.conf by the user) then we should add
it and notify the user, otherwise devtool operations won't work.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/devtool | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/scripts/devtool b/scripts/devtool
index 841831c..0100eb8 100755
--- a/scripts/devtool
+++ b/scripts/devtool
@@ -2,7 +2,7 @@
 
 # OpenEmbedded Development tool
 #
-# Copyright (C) 2014 Intel Corporation
+# Copyright (C) 2014-2015 Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -99,6 +99,8 @@ def read_workspace():
         else:
             logger.info('Creating workspace layer in %s' % config.workspace_path)
             _create_workspace(config.workspace_path, config, basepath)
+    if not context.fixed_setup:
+        _enable_workspace_layer(config.workspace_path, config, basepath)
 
     logger.debug('Reading workspace in %s' % config.workspace_path)
     externalsrc_re = re.compile(r'^EXTERNALSRC(_pn-[^ =]+)? =.*$')
@@ -116,9 +118,11 @@ def create_workspace(args, config, basepath, workspace):
         workspacedir = os.path.abspath(args.layerpath)
     else:
         workspacedir = os.path.abspath(os.path.join(basepath, 'workspace'))
-    _create_workspace(workspacedir, config, basepath, args.create_only)
+    _create_workspace(workspacedir, config, basepath)
+    if not args.create_only:
+        _enable_workspace_layer(workspacedir, config, basepath)
 
-def _create_workspace(workspacedir, config, basepath, create_only=False):
+def _create_workspace(workspacedir, config, basepath):
     import bb
 
     confdir = os.path.join(workspacedir, 'conf')
@@ -146,17 +150,21 @@ def _create_workspace(workspacedir, config, basepath, create_only=False):
             f.write('\nIf you no longer need to use devtool you can remove the path to this\n')
             f.write('workspace layer from your conf/bblayers.conf file (and then delete the\n')
             f.write('layer, if you wish).\n')
-    if not create_only:
-        # Add the workspace layer to bblayers.conf
-        bblayers_conf = os.path.join(basepath, 'conf', 'bblayers.conf')
-        if not os.path.exists(bblayers_conf):
-            logger.error('Unable to find bblayers.conf')
-            return -1
-        bb.utils.edit_bblayers_conf(bblayers_conf, workspacedir, config.workspace_path)
-        if config.workspace_path != workspacedir:
-            # Update our config to point to the new location
-            config.workspace_path = workspacedir
-            config.write()
+
+def _enable_workspace_layer(workspacedir, config, basepath):
+    """Ensure the workspace layer is in bblayers.conf"""
+    import bb
+    bblayers_conf = os.path.join(basepath, 'conf', 'bblayers.conf')
+    if not os.path.exists(bblayers_conf):
+        logger.error('Unable to find bblayers.conf')
+        return -1
+    _, added = bb.utils.edit_bblayers_conf(bblayers_conf, workspacedir, config.workspace_path)
+    if added:
+        logger.info('Enabling workspace layer in bblayers.conf')
+    if config.workspace_path != workspacedir:
+        # Update our config to point to the new location
+        config.workspace_path = workspacedir
+        config.write()
 
 
 def main():
-- 
2.1.0