[PATCH] devtool: include bbappends in recipe parsing

[PATCH] devtool: include bbappends in recipe parsing

[Markus Lehtonen]

In order to get correct metadata, SRCREV for example.

Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
 meta/lib/oe/recipeutils.py      |  6 +++---
 scripts/lib/devtool/standard.py | 32 +++++++++++++++++++-------------
 2 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index 159a103..09bd7fd 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -44,10 +44,10 @@ def get_unavailable_reasons(cooker, pn):
     return taskdata.get_reasons(pn)
 
 
-def parse_recipe(fn, d):
+def parse_recipe(fn, appends, d):
     """Parse an individual recipe"""
     import bb.cache
-    envdata = bb.cache.Cache.loadDataFull(fn, [], d)
+    envdata = bb.cache.Cache.loadDataFull(fn, appends, d)
     return envdata
 
 
@@ -55,7 +55,7 @@ def get_var_files(fn, varlist, d):
     """Find the file in which each of a list of variables is set.
     Note: requires variable history to be enabled when parsing.
     """
-    envdata = parse_recipe(fn, d)
+    envdata = parse_recipe(fn, [], d)
     varfiles = {}
     for v in varlist:
         history = envdata.varhistory.variable(v)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index d5ded2f..96e3bdb 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -130,18 +130,26 @@ def _get_recipe_file(cooker, pn):
             logger.error("Unable to find any recipe file matching %s" % pn)
     return recipefile
 
+def _parse_recipe(tinfoil, pn, appends):
+    """Parse recipe of a package"""
+    import oe.recipeutils
+    recipefile = _get_recipe_file(tinfoil.cooker, pn)
+    if not recipefile:
+        # Error already logged
+        return None
+    if appends:
+        append_files = tinfoil.cooker.collection.get_file_appends(recipefile)
+    return oe.recipeutils.parse_recipe(recipefile, append_files,
+                                       tinfoil.config_data)
 
 def extract(args, config, basepath, workspace):
     import bb
-    import oe.recipeutils
 
     tinfoil = setup_tinfoil()
 
-    recipefile = _get_recipe_file(tinfoil.cooker, args.recipename)
-    if not recipefile:
-        # Error already logged
+    rd = _parse_recipe(tinfoil, args.recipename, True)
+    if not rd:
         return -1
-    rd = oe.recipeutils.parse_recipe(recipefile, tinfoil.config_data)
 
     srctree = os.path.abspath(args.srctree)
     initial_rev = _extract_source(srctree, args.keep_temp, args.branch, rd)
@@ -327,11 +335,10 @@ def modify(args, config, basepath, workspace):
 
     tinfoil = setup_tinfoil()
 
-    recipefile = _get_recipe_file(tinfoil.cooker, args.recipename)
-    if not recipefile:
-        # Error already logged
+    rd = _parse_recipe(tinfoil, args.recipename, True)
+    if not rd:
         return -1
-    rd = oe.recipeutils.parse_recipe(recipefile, tinfoil.config_data)
+    recipefile = rd.getVar('FILE', True)
 
     if not _check_compatible_recipe(args.recipename, rd):
         return -1
@@ -418,11 +425,10 @@ def update_recipe(args, config, basepath, workspace):
     from oe.patch import GitApplyTree
     import oe.recipeutils
 
-    recipefile = _get_recipe_file(tinfoil.cooker, args.recipename)
-    if not recipefile:
-        # Error already logged
+    rd = _parse_recipe(tinfoil, args.recipename, True)
+    if not rd:
         return -1
-    rd = oe.recipeutils.parse_recipe(recipefile, tinfoil.config_data)
+    recipefile = rd.getVar('FILE', True)
 
     orig_src_uri = rd.getVar('SRC_URI', False) or ''
     if args.mode == 'auto':
-- 
2.1.4

Re: [PATCH] devtool: include bbappends in recipe parsing

[Paul Eggleton]

On Tuesday 21 April 2015 17:50:43 Markus Lehtonen wrote:
> In order to get correct metadata, SRCREV for example.
> 
> Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
> ---
>  meta/lib/oe/recipeutils.py      |  6 +++---
>  scripts/lib/devtool/standard.py | 32 +++++++++++++++++++-------------
>  2 files changed, 22 insertions(+), 16 deletions(-)
> 
> diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
> index 159a103..09bd7fd 100644
> --- a/meta/lib/oe/recipeutils.py
> +++ b/meta/lib/oe/recipeutils.py
> @@ -44,10 +44,10 @@ def get_unavailable_reasons(cooker, pn):
>      return taskdata.get_reasons(pn)
> 
> 
> -def parse_recipe(fn, d):
> +def parse_recipe(fn, appends, d):
>      """Parse an individual recipe"""
>      import bb.cache
> -    envdata = bb.cache.Cache.loadDataFull(fn, [], d)
> +    envdata = bb.cache.Cache.loadDataFull(fn, appends, d)
>      return envdata
> 
> 
> @@ -55,7 +55,7 @@ def get_var_files(fn, varlist, d):
>      """Find the file in which each of a list of variables is set.
>      Note: requires variable history to be enabled when parsing.
>      """
> -    envdata = parse_recipe(fn, d)
> +    envdata = parse_recipe(fn, [], d)
>      varfiles = {}
>      for v in varlist:
>          history = envdata.varhistory.variable(v)
> diff --git a/scripts/lib/devtool/standard.py
> b/scripts/lib/devtool/standard.py index d5ded2f..96e3bdb 100644
> --- a/scripts/lib/devtool/standard.py
> +++ b/scripts/lib/devtool/standard.py
> @@ -130,18 +130,26 @@ def _get_recipe_file(cooker, pn):
>              logger.error("Unable to find any recipe file matching %s" % pn)
> return recipefile
> 
> +def _parse_recipe(tinfoil, pn, appends):
> +    """Parse recipe of a package"""
> +    import oe.recipeutils
> +    recipefile = _get_recipe_file(tinfoil.cooker, pn)
> +    if not recipefile:
> +        # Error already logged
> +        return None
> +    if appends:
> +        append_files =
> tinfoil.cooker.collection.get_file_appends(recipefile) +    return
> oe.recipeutils.parse_recipe(recipefile, append_files,
> +                                       tinfoil.config_data)
> 
>  def extract(args, config, basepath, workspace):
>      import bb
> -    import oe.recipeutils
> 
>      tinfoil = setup_tinfoil()
> 
> -    recipefile = _get_recipe_file(tinfoil.cooker, args.recipename)
> -    if not recipefile:
> -        # Error already logged
> +    rd = _parse_recipe(tinfoil, args.recipename, True)
> +    if not rd:
>          return -1
> -    rd = oe.recipeutils.parse_recipe(recipefile, tinfoil.config_data)
> 
>      srctree = os.path.abspath(args.srctree)
>      initial_rev = _extract_source(srctree, args.keep_temp, args.branch, rd)
> @@ -327,11 +335,10 @@ def modify(args, config, basepath, workspace):
> 
>      tinfoil = setup_tinfoil()
> 
> -    recipefile = _get_recipe_file(tinfoil.cooker, args.recipename)
> -    if not recipefile:
> -        # Error already logged
> +    rd = _parse_recipe(tinfoil, args.recipename, True)
> +    if not rd:
>          return -1
> -    rd = oe.recipeutils.parse_recipe(recipefile, tinfoil.config_data)
> +    recipefile = rd.getVar('FILE', True)
> 
>      if not _check_compatible_recipe(args.recipename, rd):
>          return -1
> @@ -418,11 +425,10 @@ def update_recipe(args, config, basepath, workspace):
>      from oe.patch import GitApplyTree
>      import oe.recipeutils
> 
> -    recipefile = _get_recipe_file(tinfoil.cooker, args.recipename)
> -    if not recipefile:
> -        # Error already logged
> +    rd = _parse_recipe(tinfoil, args.recipename, True)
> +    if not rd:
>          return -1
> -    rd = oe.recipeutils.parse_recipe(recipefile, tinfoil.config_data)
> +    recipefile = rd.getVar('FILE', True)
> 
>      orig_src_uri = rd.getVar('SRC_URI', False) or ''
>      if args.mode == 'auto':

I ran the oe-selftest tests and unfortunately this appears to cause a 
regression - you can see the test failure by running:

oe-selftest --run-tests devtool.DevtoolTests.test_devtool_update_recipe_git

I suspect this might be because with this patch we'd be seeing the bbappends 
within the workspace layer, which might tangle things up.

Cheers,
Paul

-- 

Paul Eggleton
Intel Open Source Technology Centre

Re: [PATCH] devtool: include bbappends in recipe parsing

[Markus Lehtonen]

Hi,


On Tue, 2015-04-21 at 17:42 +0100, Paul Eggleton wrote:
> On Tuesday 21 April 2015 17:50:43 Markus Lehtonen wrote:
> > In order to get correct metadata, SRCREV for example.
> > 
> > Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
> > ---
> >  meta/lib/oe/recipeutils.py      |  6 +++---
> >  scripts/lib/devtool/standard.py | 32 +++++++++++++++++++-------------
> >  2 files changed, 22 insertions(+), 16 deletions(-)
> > 
> > diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
> > index 159a103..09bd7fd 100644
> > --- a/meta/lib/oe/recipeutils.py
> > +++ b/meta/lib/oe/recipeutils.py
> > @@ -44,10 +44,10 @@ def get_unavailable_reasons(cooker, pn):
> >      return taskdata.get_reasons(pn)
> > 
> > 
> > -def parse_recipe(fn, d):
> > +def parse_recipe(fn, appends, d):
> >      """Parse an individual recipe"""
> >      import bb.cache
> > -    envdata = bb.cache.Cache.loadDataFull(fn, [], d)
> > +    envdata = bb.cache.Cache.loadDataFull(fn, appends, d)
> >      return envdata
> > 
> > 
> > @@ -55,7 +55,7 @@ def get_var_files(fn, varlist, d):
> >      """Find the file in which each of a list of variables is set.
> >      Note: requires variable history to be enabled when parsing.
> >      """
> > -    envdata = parse_recipe(fn, d)
> > +    envdata = parse_recipe(fn, [], d)
> >      varfiles = {}
> >      for v in varlist:
> >          history = envdata.varhistory.variable(v)
> > diff --git a/scripts/lib/devtool/standard.py
> > b/scripts/lib/devtool/standard.py index d5ded2f..96e3bdb 100644
> > --- a/scripts/lib/devtool/standard.py
> > +++ b/scripts/lib/devtool/standard.py
> > @@ -130,18 +130,26 @@ def _get_recipe_file(cooker, pn):
> >              logger.error("Unable to find any recipe file matching %s" % pn)
> > return recipefile
> > 
> > +def _parse_recipe(tinfoil, pn, appends):
> > +    """Parse recipe of a package"""
> > +    import oe.recipeutils
> > +    recipefile = _get_recipe_file(tinfoil.cooker, pn)
> > +    if not recipefile:
> > +        # Error already logged
> > +        return None
> > +    if appends:
> > +        append_files =
> > tinfoil.cooker.collection.get_file_appends(recipefile) +    return
> > oe.recipeutils.parse_recipe(recipefile, append_files,
> > +                                       tinfoil.config_data)
> > 
> >  def extract(args, config, basepath, workspace):
> >      import bb
> > -    import oe.recipeutils
> > 
> >      tinfoil = setup_tinfoil()
> > 
> > -    recipefile = _get_recipe_file(tinfoil.cooker, args.recipename)
> > -    if not recipefile:
> > -        # Error already logged
> > +    rd = _parse_recipe(tinfoil, args.recipename, True)
> > +    if not rd:
> >          return -1
> > -    rd = oe.recipeutils.parse_recipe(recipefile, tinfoil.config_data)
> > 
> >      srctree = os.path.abspath(args.srctree)
> >      initial_rev = _extract_source(srctree, args.keep_temp, args.branch, rd)
> > @@ -327,11 +335,10 @@ def modify(args, config, basepath, workspace):
> > 
> >      tinfoil = setup_tinfoil()
> > 
> > -    recipefile = _get_recipe_file(tinfoil.cooker, args.recipename)
> > -    if not recipefile:
> > -        # Error already logged
> > +    rd = _parse_recipe(tinfoil, args.recipename, True)
> > +    if not rd:
> >          return -1
> > -    rd = oe.recipeutils.parse_recipe(recipefile, tinfoil.config_data)
> > +    recipefile = rd.getVar('FILE', True)
> > 
> >      if not _check_compatible_recipe(args.recipename, rd):
> >          return -1
> > @@ -418,11 +425,10 @@ def update_recipe(args, config, basepath, workspace):
> >      from oe.patch import GitApplyTree
> >      import oe.recipeutils
> > 
> > -    recipefile = _get_recipe_file(tinfoil.cooker, args.recipename)
> > -    if not recipefile:
> > -        # Error already logged
> > +    rd = _parse_recipe(tinfoil, args.recipename, True)
> > +    if not rd:
> >          return -1
> > -    rd = oe.recipeutils.parse_recipe(recipefile, tinfoil.config_data)
> > +    recipefile = rd.getVar('FILE', True)
> > 
> >      orig_src_uri = rd.getVar('SRC_URI', False) or ''
> >      if args.mode == 'auto':
> 
> I ran the oe-selftest tests and unfortunately this appears to cause a 
> regression - you can see the test failure by running:
> 
> oe-selftest --run-tests devtool.DevtoolTests.test_devtool_update_recipe_git
> 
> I suspect this might be because with this patch we'd be seeing the bbappends 
> within the workspace layer, which might tangle things up.

Good point, I'll post an updated patch.

However, seems that the tests got already broken earlier, even before my
patch. I'll send another patch to fix the tests.

Thanks,
  Markus