[PATCH 0/4] recipetool create improvements

[PATCH 0/4] recipetool create improvements

[Paul Eggleton]

Support for creating recipes for kernels, out-of-tree kernel modules,
and npm modules.


The following changes since commit 390bad905537820f49add855c95d726b5b55c8fa:

  sanity: Fix int verses string reference (2016-02-28 22:54:06 +0000)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib paule/recipetool2
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/recipetool2

Paul Eggleton (4):
  classes/module: allow substitution of the modules_install target name
  recipetool: create: add support for out-of-tree kernel modules
  recipetool: create: add basic support for generating linux kernel recipes
  recipetool: create: add basic support for new npm fetcher/class

 meta/classes/module.bbclass             |   4 +-
 scripts/lib/recipetool/create_kernel.py |  99 +++++++++++++++++++++
 scripts/lib/recipetool/create_kmod.py   | 152 ++++++++++++++++++++++++++++++++
 scripts/lib/recipetool/create_npm.py    |  48 ++++++++++
 4 files changed, 302 insertions(+), 1 deletion(-)
 create mode 100644 scripts/lib/recipetool/create_kernel.py
 create mode 100644 scripts/lib/recipetool/create_kmod.py
 create mode 100644 scripts/lib/recipetool/create_npm.py

-- 
2.5.0

[PATCH 1/4] classes/module: allow substitution of the modules_install target name

[Paul Eggleton]

Quite a few external kernel modules I've found floating around don't
have a modules_install target, but they do have an install target that
basically differs only in name. To make it easier to build these just
make this a MODULES_INSTALL_TARGET variable that you can set from the
recipe - the alternative would be copy-and-paste the do_install
definition from this class which is potentially fragile.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 meta/classes/module.bbclass | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/meta/classes/module.bbclass b/meta/classes/module.bbclass
index 0952c0c..01c9309 100644
--- a/meta/classes/module.bbclass
+++ b/meta/classes/module.bbclass
@@ -6,6 +6,8 @@ do_make_scripts[depends] += "virtual/kernel:do_shared_workdir"
 
 EXTRA_OEMAKE += "KERNEL_SRC=${STAGING_KERNEL_DIR}"
 
+MODULES_INSTALL_TARGET ?= "modules_install"
+
 module_do_compile() {
 	unset CFLAGS CPPFLAGS CXXFLAGS LDFLAGS
 	oe_runmake KERNEL_PATH=${STAGING_KERNEL_DIR}   \
@@ -21,7 +23,7 @@ module_do_install() {
 	oe_runmake DEPMOD=echo INSTALL_MOD_PATH="${D}" \
 	           CC="${KERNEL_CC}" LD="${KERNEL_LD}" \
 	           O=${STAGING_KERNEL_BUILDDIR} \
-	           modules_install
+	           ${MODULES_INSTALL_TARGET}
 }
 
 EXPORT_FUNCTIONS do_compile do_install
-- 
2.5.0

[PATCH 2/4] recipetool: create: add support for out-of-tree kernel modules

[Paul Eggleton]

Detect kernel modules by looking for #include <linux/module.h>, and
handle the various styles of Makefile that appear to be used. I was able
to use this code to successfully build a number of external kernel
modules I found.

Implements [YOCTO #8982].

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/recipetool/create_kmod.py | 152 ++++++++++++++++++++++++++++++++++
 1 file changed, 152 insertions(+)
 create mode 100644 scripts/lib/recipetool/create_kmod.py

diff --git a/scripts/lib/recipetool/create_kmod.py b/scripts/lib/recipetool/create_kmod.py
new file mode 100644
index 0000000..fe39edb
--- /dev/null
+++ b/scripts/lib/recipetool/create_kmod.py
@@ -0,0 +1,152 @@
+# Recipe creation tool - kernel module support plugin
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import re
+import logging
+from recipetool.create import RecipeHandler, read_pkgconfig_provides, validate_pv
+
+logger = logging.getLogger('recipetool')
+
+tinfoil = None
+
+def tinfoil_init(instance):
+    global tinfoil
+    tinfoil = instance
+
+
+class KernelModuleRecipeHandler(RecipeHandler):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+        import bb.process
+        if 'buildsystem' in handled:
+            return False
+
+        module_inc_re = re.compile(r'^#include\s+<linux/module.h>$')
+        makefiles = []
+        is_module = False
+
+        makefiles = []
+
+        files = RecipeHandler.checkfiles(srctree, ['*.c', '*.h'], recursive=True)
+        if files:
+            for cfile in files:
+                # Look in same dir or parent for Makefile
+                for makefile in [os.path.join(os.path.dirname(cfile), 'Makefile'), os.path.join(os.path.dirname(os.path.dirname(cfile)), 'Makefile')]:
+                    if makefile in makefiles:
+                        break
+                    else:
+                        if os.path.exists(makefile):
+                            makefiles.append(makefile)
+                            break
+                else:
+                    continue
+                with open(cfile, 'r') as f:
+                    for line in f:
+                        if module_inc_re.match(line.strip()):
+                            is_module = True
+                            break
+                if is_module:
+                    break
+
+        if is_module:
+            classes.append('module')
+            handled.append('buildsystem')
+            # module.bbclass and the classes it inherits do most of the hard
+            # work, but we need to tweak it slightly depending on what the
+            # Makefile does (and there is a range of those)
+            # Check the makefile for the appropriate install target
+            install_lines = []
+            compile_lines = []
+            in_install = False
+            in_compile = False
+            install_target = None
+            with open(makefile, 'r') as f:
+                for line in f:
+                    if line.startswith('install:'):
+                        if not install_lines:
+                            in_install = True
+                            install_target = 'install'
+                    elif line.startswith('modules_install:'):
+                        install_lines = []
+                        in_install = True
+                        install_target = 'modules_install'
+                    elif line.startswith('modules:'):
+                        compile_lines = []
+                        in_compile = True
+                    elif line.startswith(('all:', 'default:')):
+                        if not compile_lines:
+                            in_compile = True
+                    elif line:
+                        if line[0] == '\t':
+                            if in_install:
+                                install_lines.append(line)
+                            elif in_compile:
+                                compile_lines.append(line)
+                        elif ':' in line:
+                            in_install = False
+                            in_compile = False
+
+            def check_target(lines, install):
+                kdirpath = ''
+                manual_install = False
+                for line in lines:
+                    splitline = line.split()
+                    if splitline[0] in ['make', 'gmake', '$(MAKE)']:
+                        if '-C' in splitline:
+                            idx = splitline.index('-C') + 1
+                            if idx < len(splitline):
+                                kdirpath = splitline[idx]
+                                break
+                    elif install and splitline[0] == 'install':
+                        if '.ko' in line:
+                            manual_install = True
+                return kdirpath, manual_install
+
+            kdirpath = None
+            manual_install = False
+            if install_lines:
+                kdirpath, manual_install = check_target(install_lines, install=True)
+            if compile_lines and not kdirpath:
+                kdirpath, _ = check_target(compile_lines, install=False)
+
+            if manual_install or not install_lines:
+                lines_after.append('EXTRA_OEMAKE_append_task-install = " -C ${STAGING_KERNEL_DIR} M=${S}"')
+            elif install_target and install_target != 'modules_install':
+                lines_after.append('MODULES_INSTALL_TARGET = "install"')
+
+            warnmsg = None
+            kdirvar = None
+            if kdirpath:
+                res = re.match(r'\$\(([^$)]+)\)', kdirpath)
+                if res:
+                    kdirvar = res.group(1)
+                    if kdirvar != 'KERNEL_SRC':
+                        lines_after.append('EXTRA_OEMAKE += "%s=${STAGING_KERNEL_DIR}"' % kdirvar)
+                elif kdirpath.startswith('/lib/'):
+                    warnmsg = 'Kernel path in install makefile is hardcoded - you will need to patch the makefile'
+            if not kdirvar and not warnmsg:
+                warnmsg = 'Unable to find means of passing kernel path into install makefile - if kernel path is hardcoded you will need to patch the makefile'
+            if warnmsg:
+                warnmsg += '. Note that the variable KERNEL_SRC will be passed in as the kernel source path.'
+                logger.warn(warnmsg)
+                lines_after.append('# %s' % warnmsg)
+
+            return True
+
+        return False
+
+def register_recipe_handlers(handlers):
+    handlers.append((KernelModuleRecipeHandler(), 15))
-- 
2.5.0

[PATCH 3/4] recipetool: create: add basic support for generating linux kernel recipes

[Paul Eggleton]

Add support for detecting a Linux kernel source tree and generating a
basic kernel recipe using meta-skeleton's linux-yocto-custom recipe as a
base.

Implements [YOCTO #8981].

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/recipetool/create_kernel.py | 99 +++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)
 create mode 100644 scripts/lib/recipetool/create_kernel.py

diff --git a/scripts/lib/recipetool/create_kernel.py b/scripts/lib/recipetool/create_kernel.py
new file mode 100644
index 0000000..c6e86bd
--- /dev/null
+++ b/scripts/lib/recipetool/create_kernel.py
@@ -0,0 +1,99 @@
+# Recipe creation tool - kernel support plugin
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import re
+import logging
+from recipetool.create import RecipeHandler, read_pkgconfig_provides, validate_pv
+
+logger = logging.getLogger('recipetool')
+
+tinfoil = None
+
+def tinfoil_init(instance):
+    global tinfoil
+    tinfoil = instance
+
+
+class KernelRecipeHandler(RecipeHandler):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+        import bb.process
+        if 'buildsystem' in handled:
+            return False
+
+        for tell in ['arch', 'firmware', 'Kbuild', 'Kconfig']:
+            if not os.path.exists(os.path.join(srctree, tell)):
+                return False
+
+        handled.append('buildsystem')
+        del lines_after[:]
+        del classes[:]
+        template = os.path.join(tinfoil.config_data.getVar('COREBASE', True), 'meta-skeleton', 'recipes-kernel', 'linux', 'linux-yocto-custom.bb')
+        def handle_var(varname, origvalue, op, newlines):
+            if varname in ['SRCREV', 'SRCREV_machine']:
+                while newlines[-1].startswith('#'):
+                    del newlines[-1]
+                try:
+                    stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree, shell=True)
+                except bb.process.ExecutionError as e:
+                    stdout = None
+                if stdout:
+                    return stdout.strip(), op, 0, True
+            elif varname == 'LINUX_VERSION':
+                makefile = os.path.join(srctree, 'Makefile')
+                if os.path.exists(makefile):
+                    kversion = -1
+                    kpatchlevel = -1
+                    ksublevel = -1
+                    kextraversion = ''
+                    with open(makefile, 'r') as f:
+                        for i, line in enumerate(f):
+                            if i > 10:
+                                break
+                            if line.startswith('VERSION ='):
+                                kversion = int(line.split('=')[1].strip())
+                            elif line.startswith('PATCHLEVEL ='):
+                                kpatchlevel = int(line.split('=')[1].strip())
+                            elif line.startswith('SUBLEVEL ='):
+                                ksublevel = int(line.split('=')[1].strip())
+                            elif line.startswith('EXTRAVERSION ='):
+                                kextraversion = line.split('=')[1].strip()
+                    version = ''
+                    if kversion > -1 and kpatchlevel > -1:
+                        version = '%d.%d' % (kversion, kpatchlevel)
+                        if ksublevel > -1:
+                            version += '.%d' % ksublevel
+                        version += kextraversion
+                    if version:
+                        return version, op, 0, True
+            elif varname == 'SRC_URI':
+                while newlines[-1].startswith('#'):
+                    del newlines[-1]
+            elif varname == 'COMPATIBLE_MACHINE':
+                while newlines[-1].startswith('#'):
+                    del newlines[-1]
+                machine = tinfoil.config_data.getVar('MACHINE', True)
+                return machine, op, 0, True
+            return origvalue, op, 0, True
+        with open(template, 'r') as f:
+            varlist = ['SRCREV', 'SRCREV_machine', 'SRC_URI', 'LINUX_VERSION', 'COMPATIBLE_MACHINE']
+            (_, newlines) = bb.utils.edit_metadata(f, varlist, handle_var)
+        lines_before[:] = [line.rstrip('\n') for line in newlines]
+
+        return True
+
+def register_recipe_handlers(handlers):
+    handlers.append((KernelRecipeHandler(), 100))
-- 
2.5.0

[PATCH 4/4] recipetool: create: add basic support for new npm fetcher/class

[Paul Eggleton]

Add detection for npm modules and support for extracting the name and
version from package.json as is usually part of an npm module contents.

Note: this will likely only produce a buildable recipe if you use an
npm:// URL; simply pointing to a node.js source repository isn't going
to fetch the module's dependencies. It also doesn't set up the
shrinkwrap/lockdown automatically, so there is some room for improvement
later.

Implements [YOCTO #8690].

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/recipetool/create_npm.py | 48 ++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 scripts/lib/recipetool/create_npm.py

diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
new file mode 100644
index 0000000..0e33cc9
--- /dev/null
+++ b/scripts/lib/recipetool/create_npm.py
@@ -0,0 +1,48 @@
+# Recipe creation tool - node.js NPM module support plugin
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import logging
+import json
+from recipetool.create import RecipeHandler
+
+logger = logging.getLogger('recipetool')
+
+
+class NpmRecipeHandler(RecipeHandler):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+        if 'buildsystem' in handled:
+            return False
+
+        files = RecipeHandler.checkfiles(srctree, ['package.json'])
+        if files:
+            with open(files[0], 'r') as f:
+                data = json.loads(f.read())
+            if 'name' in data and 'version' in data:
+                extravalues['PN'] = data['name']
+                extravalues['PV'] = data['version']
+                classes.append('npm')
+                handled.append('buildsystem')
+                if 'description' in data:
+                    lines_before.append('SUMMARY = "%s"' % data['description'])
+                if 'homepage' in data:
+                    lines_before.append('HOMEPAGE = "%s"' % data['homepage'])
+                return True
+
+        return False
+
+def register_recipe_handlers(handlers):
+    handlers.append((NpmRecipeHandler(), 60))
-- 
2.5.0