[PATCH 0/3] devtool: improvements to kernel support

[PATCH 0/3] devtool: improvements to kernel support

[Markus Lehtonen]

This patchset contains few patches to slightly improve the support for kernel
packages in devtool. The last of which is an ugly hack - to be removed after a
future tinfoil rewrite when running arbitrary bitbake task functions becomes
possible.

Markus Lehtonen (3):
  devtool: make required tasks be run in kernel build
  devtool: extract: correct initial rev for kernel packages
  devtool: run kernel dependencies

 scripts/lib/devtool/standard.py | 54 +++++++++++++++++++++++++++--------------
 1 file changed, 36 insertions(+), 18 deletions(-)

-- 
2.1.4

[PATCH 1/3] devtool: make required tasks be run in kernel build

[Markus Lehtonen]

Set SRCTREECOVEREDTASKS appropriately in the workspace .bbappend file
for kernel recipes. This tries to ensure that all needed tasks (esp.
configure and patch) are run when building the kernel - tasks which
would normally be disabled by externalsrc.bbclass.

[YOCTO #6658]

Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
 scripts/lib/devtool/standard.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index e1c5584..00d0b2c 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -500,6 +500,8 @@ def modify(args, config, basepath, workspace):
         if b_is_s:
             f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (args.recipename, srctree))
 
+        if bb.data.inherits_class('kernel', rd):
+            f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout do_shared_workdir do_fetch do_unpack"\n')
         if initial_rev:
             f.write('\n# initial_rev: %s\n' % initial_rev)
             for commit in commits:
-- 
2.1.4

[PATCH 2/3] devtool: extract: correct initial rev for kernel packages

[Markus Lehtonen]

Change handling of kernel packages so that the "initial rev" is parsed
correctly. Also, the devtool-specific git tags (devtool-base and
devtoo-patched) are now generated for kernel packages as well.

[YOCTO #6658]

Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
 scripts/lib/devtool/standard.py | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 00d0b2c..3725d87 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -300,6 +300,10 @@ def _extract_source(srctree, keep_temp, devbranch, d):
         task_executor.exec_func('do_fetch', False)
         logger.info('Unpacking...')
         task_executor.exec_func('do_unpack', False)
+        if bb.data.inherits_class('kernel-yocto', d):
+            # Extra step for kernel to populate the source directory
+            logger.info('Doing kernel checkout...')
+            task_executor.exec_func('do_kernel_checkout', False)
         srcsubdir = crd.getVar('S', True)
         if srcsubdir == workdir:
             # Find non-patch sources that were "unpacked" to srctree directory
@@ -329,26 +333,21 @@ def _extract_source(srctree, keep_temp, devbranch, d):
             else:
                 os.rmdir(patchdir)
 
-        if bb.data.inherits_class('kernel-yocto', d):
-            (stdout, _) = bb.process.run('git --git-dir="%s" rev-parse HEAD' % crd.expand('${WORKDIR}/git'), cwd=srcsubdir)
-            initial_rev = stdout.rstrip()
-        else:
-            if not os.listdir(srcsubdir):
-                raise DevtoolError("no source unpacked to S, perhaps the %s "
-                                   "recipe doesn't use any source?" % pn)
+        if not os.listdir(srcsubdir):
+            raise DevtoolError("no source unpacked to S, perhaps the %s "
+                               "recipe doesn't use any source?" % pn)
 
-            if not os.path.exists(os.path.join(srcsubdir, '.git')):
-                bb.process.run('git init', cwd=srcsubdir)
-                bb.process.run('git add .', cwd=srcsubdir)
-                bb.process.run('git commit -q -m "Initial commit from upstream at version %s"' % crd.getVar('PV', True), cwd=srcsubdir)
+        if not os.path.exists(os.path.join(srcsubdir, '.git')):
+            bb.process.run('git init', cwd=srcsubdir)
+            bb.process.run('git add .', cwd=srcsubdir)
+            bb.process.run('git commit -q -m "Initial commit from upstream at version %s"' % crd.getVar('PV', True), cwd=srcsubdir)
 
-            (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
-            initial_rev = stdout.rstrip()
-
-            bb.process.run('git checkout -b %s' % devbranch, cwd=srcsubdir)
-            bb.process.run('git tag -f devtool-base', cwd=srcsubdir)
+        (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
+        initial_rev = stdout.rstrip()
 
-            crd.setVar('PATCHTOOL', 'git')
+        bb.process.run('git checkout -b %s' % devbranch, cwd=srcsubdir)
+        bb.process.run('git tag -f devtool-base', cwd=srcsubdir)
+        crd.setVar('PATCHTOOL', 'git')
 
         logger.info('Patching...')
         task_executor.exec_func('do_patch', False)
-- 
2.1.4

[PATCH 3/3] devtool: run kernel dependencies

[Markus Lehtonen]

The kernel package needs "kern-tools-native" in order for it's
do_kernel_metadata. Thus, devtool extract for kernel in a pristine
environment fails. With the current bb.tinfoil implementation it is not
possible to run arbitrary bitbake commands - e.g. run
"bitbake kern-tools-native -c populate_sysroot" in our case. This patch
implements an ugly workaround for that problem, basically by hardcoding
this dependency and running the required bitbake task(s) before tinfoil
is initialized.

[YOCTO #6658]

Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
 scripts/lib/devtool/standard.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 3725d87..bf24e32 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -197,6 +197,8 @@ def extract(args, config, basepath, workspace):
     """Entry point for the devtool 'extract' subcommand"""
     import bb
 
+    _check_extract_deps(config, basepath, args.recipename)
+
     tinfoil = setup_tinfoil()
 
     rd = _parse_recipe(config, tinfoil, args.recipename, True)
@@ -237,6 +239,20 @@ class BbTaskExecutor(object):
             self.executed.append(func)
 
 
+def _check_extract_deps(config, basepath, recipename):
+    """HACK: Ugly workaround for making sure that requirements are met when
+       trying to extract a package"""
+    tinfoil = setup_tinfoil()
+    rd = _parse_recipe(config, tinfoil, recipename, True)
+    if bb.data.inherits_class('kernel-yocto', rd):
+        tinfoil.shutdown()
+        try:
+            stdout, _ = exec_build_env_command(config.init_path, basepath,
+                                               'bitbake kern-tools-native')
+        except bb.process.ExecutionError as err:
+            raise DevtoolError("Failed to build kern-tools-native:\n%s" %
+                               err.stdout)
+
 def _extract_source(srctree, keep_temp, devbranch, d):
     """Extract sources of a recipe"""
     import bb.event
@@ -422,7 +438,8 @@ def modify(args, config, basepath, workspace):
         raise DevtoolError("directory %s does not exist or not a directory "
                            "(specify -x to extract source from recipe)" %
                            args.srctree)
-
+    if args.extract:
+        _check_extract_deps(config, basepath, args.recipename)
     tinfoil = setup_tinfoil()
 
     rd = _parse_recipe(config, tinfoil, args.recipename, True)
-- 
2.1.4

Re: [PATCH 3/3] devtool: run kernel dependencies

[Paul Eggleton]

Hi Markus,

On Thursday 27 August 2015 14:49:50 Markus Lehtonen wrote:
> The kernel package needs "kern-tools-native" in order for it's
> do_kernel_metadata. Thus, devtool extract for kernel in a pristine
> environment fails. With the current bb.tinfoil implementation it is not
> possible to run arbitrary bitbake commands - e.g. run
> "bitbake kern-tools-native -c populate_sysroot" in our case. This patch
> implements an ugly workaround for that problem, basically by hardcoding
> this dependency and running the required bitbake task(s) before tinfoil
> is initialized.
> 
> [YOCTO #6658]
> 
> Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
> ---
>  scripts/lib/devtool/standard.py | 19 ++++++++++++++++++-
>  1 file changed, 18 insertions(+), 1 deletion(-)
> 
> diff --git a/scripts/lib/devtool/standard.py
> b/scripts/lib/devtool/standard.py index 3725d87..bf24e32 100644
> --- a/scripts/lib/devtool/standard.py
> +++ b/scripts/lib/devtool/standard.py
> @@ -197,6 +197,8 @@ def extract(args, config, basepath, workspace):
>      """Entry point for the devtool 'extract' subcommand"""
>      import bb
> 
> +    _check_extract_deps(config, basepath, args.recipename)
> +
>      tinfoil = setup_tinfoil()
> 
>      rd = _parse_recipe(config, tinfoil, args.recipename, True)
> @@ -237,6 +239,20 @@ class BbTaskExecutor(object):
>              self.executed.append(func)
> 
> 
> +def _check_extract_deps(config, basepath, recipename):
> +    """HACK: Ugly workaround for making sure that requirements are met when
> +       trying to extract a package"""
> +    tinfoil = setup_tinfoil()
> +    rd = _parse_recipe(config, tinfoil, recipename, True)
> +    if bb.data.inherits_class('kernel-yocto', rd):
> +        tinfoil.shutdown()

Hmm, surely you need to call shutdown() in the non-kernel case as well? 
Otherwise won't the lock still be in place when this function returns and then 
setup_tinfoil() is called a second time?

> +        try:
> +            stdout, _ = exec_build_env_command(config.init_path, basepath,
> +                                               'bitbake kern-tools-native')
> +        except bb.process.ExecutionError as err:
> +            raise DevtoolError("Failed to build kern-tools-native:\n%s" %
> +                               err.stdout)
> +
>  def _extract_source(srctree, keep_temp, devbranch, d):
>      """Extract sources of a recipe"""
>      import bb.event
> @@ -422,7 +438,8 @@ def modify(args, config, basepath, workspace):
>          raise DevtoolError("directory %s does not exist or not a directory
> " "(specify -x to extract source from recipe)" % args.srctree)
> -
> +    if args.extract:
> +        _check_extract_deps(config, basepath, args.recipename)
>      tinfoil = setup_tinfoil()
> 
>      rd = _parse_recipe(config, tinfoil, args.recipename, True)

Cheers,
Paul

-- 

Paul Eggleton
Intel Open Source Technology Centre

Re: [PATCH 3/3] devtool: run kernel dependencies

[Markus Lehtonen]

[-- Attachment #1: Type: text/plain, Size: 2465 bytes --]

Hi,

On 27/08/15 15:45, "Paul Eggleton" <paul.eggleton@linux.intel.com> wrote:

>Hi Markus,
>
>On Thursday 27 August 2015 14:49:50 Markus Lehtonen wrote:
>> The kernel package needs "kern-tools-native" in order for it's
>> do_kernel_metadata. Thus, devtool extract for kernel in a pristine
>> environment fails. With the current bb.tinfoil implementation it is not
>> possible to run arbitrary bitbake commands - e.g. run
>> "bitbake kern-tools-native -c populate_sysroot" in our case. This patch
>> implements an ugly workaround for that problem, basically by hardcoding
>> this dependency and running the required bitbake task(s) before tinfoil
>> is initialized.
>> 
>> [YOCTO #6658]
>> 
>> Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
>> ---
>>  scripts/lib/devtool/standard.py | 19 ++++++++++++++++++-
>>  1 file changed, 18 insertions(+), 1 deletion(-)
>> 
>> diff --git a/scripts/lib/devtool/standard.py
>> b/scripts/lib/devtool/standard.py index 3725d87..bf24e32 100644
>> --- a/scripts/lib/devtool/standard.py
>> +++ b/scripts/lib/devtool/standard.py
>> @@ -197,6 +197,8 @@ def extract(args, config, basepath, workspace):
>>      """Entry point for the devtool 'extract' subcommand"""
>>      import bb
>> 
>> +    _check_extract_deps(config, basepath, args.recipename)
>> +
>>      tinfoil = setup_tinfoil()
>> 
>>      rd = _parse_recipe(config, tinfoil, args.recipename, True)
>> @@ -237,6 +239,20 @@ class BbTaskExecutor(object):
>>              self.executed.append(func)
>> 
>> 
>> +def _check_extract_deps(config, basepath, recipename):
>> +    """HACK: Ugly workaround for making sure that requirements are met
>>when
>> +       trying to extract a package"""
>> +    tinfoil = setup_tinfoil()
>> +    rd = _parse_recipe(config, tinfoil, recipename, True)
>> +    if bb.data.inherits_class('kernel-yocto', rd):
>> +        tinfoil.shutdown()
>
>Hmm, surely you need to call shutdown() in the non-kernel case as well?
>Otherwise won't the lock still be in place when this function returns and
>then 
>setup_tinfoil() is called a second time?

Argh, I was a bit too hasty with this one. You can find a new version of
this patch attached as well as in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib
marquiz/devtool/kernel
  
http://git.openembedded.org/openembedded-core-contrib/log/?h=marquiz/devtoo
l/kernel



Thanks,
  Markus


[-- Attachment #2: 0001-devtool-run-kernel-dependencies.patch --]
[-- Type: application/octet-stream, Size: 2957 bytes --]

From c15e4c9f17cd0bf6bfcc7c18a5f034d360d670ac Mon Sep 17 00:00:00 2001
From: Markus Lehtonen <markus.lehtonen@linux.intel.com>
Date: Wed, 26 Aug 2015 17:13:46 +0300
Subject: [PATCH] devtool: run kernel dependencies

The kernel package needs "kern-tools-native" in order for it's
do_kernel_metadata. Thus, devtool extract for kernel in a pristine
environment fails. With the current bb.tinfoil implementation it is not
possible to run arbitrary bitbake commands - e.g. run
"bitbake kern-tools-native -c populate_sysroot" in our case. This patch
implements an ugly workaround for that problem, basically by hardcoding
this dependency and running the required bitbake task(s) before tinfoil
is initialized.

[YOCTO #6658]

Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
 scripts/lib/devtool/standard.py | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 3725d87..7456fc8 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -197,7 +197,7 @@ def extract(args, config, basepath, workspace):
     """Entry point for the devtool 'extract' subcommand"""
     import bb
 
-    tinfoil = setup_tinfoil()
+    tinfoil = _prep_extract_operation(config, basepath, args.recipename)
 
     rd = _parse_recipe(config, tinfoil, args.recipename, True)
     if not rd:
@@ -237,6 +237,24 @@ class BbTaskExecutor(object):
             self.executed.append(func)
 
 
+def _prep_extract_operation(config, basepath, recipename):
+    """HACK: Ugly workaround for making sure that requirements are met when
+       trying to extract a package. Returns the tinfoil instance to be used."""
+    tinfoil = setup_tinfoil()
+    rd = _parse_recipe(config, tinfoil, recipename, True)
+
+    if bb.data.inherits_class('kernel-yocto', rd):
+        tinfoil.shutdown()
+        try:
+            stdout, _ = exec_build_env_command(config.init_path, basepath,
+                                               'bitbake kern-tools-native')
+            tinfoil = setup_tinfoil()
+        except bb.process.ExecutionError as err:
+            raise DevtoolError("Failed to build kern-tools-native:\n%s" %
+                               err.stdout)
+    return tinfoil
+
+
 def _extract_source(srctree, keep_temp, devbranch, d):
     """Extract sources of a recipe"""
     import bb.event
@@ -422,8 +440,10 @@ def modify(args, config, basepath, workspace):
         raise DevtoolError("directory %s does not exist or not a directory "
                            "(specify -x to extract source from recipe)" %
                            args.srctree)
-
-    tinfoil = setup_tinfoil()
+    if args.extract:
+        tinfoil = _prep_extract_operation(config, basepath, args.recipename)
+    else:
+        tinfoil = setup_tinfoil()
 
     rd = _parse_recipe(config, tinfoil, args.recipename, True)
     if not rd:
-- 
2.1.4