* [PATCH 10/14] lib/oe/recipeutils: drop parse_recipe_simple()
From: Paul Eggleton @ 2016-12-13 7:09 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481612135.git.paul.eggleton@linux.intel.com>
This was intended to be used with tinfoil, but tinfoil now has its own
parse_recipe() method to do this which works properly in the memres
case.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
meta/lib/oe/recipeutils.py | 22 ----------------------
1 file changed, 22 deletions(-)
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index 92fa431..26c926f 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -52,28 +52,6 @@ def parse_recipe(cooker, fn, appendfiles):
return envdata
-def parse_recipe_simple(cooker, pn, d, appends=True):
- """
- Parse a recipe and optionally all bbappends that apply to it
- in the current configuration.
- """
- import bb.providers
-
- recipefile = pn_to_recipe(cooker, pn)
- if not recipefile:
- skipreasons = get_unavailable_reasons(cooker, pn)
- # We may as well re-use bb.providers.NoProvider here
- if skipreasons:
- raise bb.providers.NoProvider(skipreasons)
- else:
- raise bb.providers.NoProvider('Unable to find any recipe file matching %s' % pn)
- if appends:
- appendfiles = cooker.collection.get_file_appends(recipefile)
- else:
- appendfiles = None
- return parse_recipe(cooker, recipefile, appendfiles)
-
-
def get_var_files(fn, varlist, d):
"""Find the file in which each of a list of variables is set.
Note: requires variable history to be enabled when parsing.
--
2.5.5
^ permalink raw reply related
* [PATCH 09/14] devtool: fix extraction of source to work in memres mode
From: Paul Eggleton @ 2016-12-13 7:09 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481612135.git.paul.eggleton@linux.intel.com>
Extracting the source for a recipe (as used by devtool's extract, modify
and upgrade subcommands) requires us to run do_fetch, do_unpack,
do_patch and any tasks that the recipe has inserted inbetween, and do so
with a modified datastore primarily so that we can redirect WORKDIR and
STAMPS_DIR in order to have the files written out to a place of our
choosing and avoid stamping the tasks as having executed in a real build
context respectively. However, this all gets much more difficult when in
memres mode since we can't call internal functions such as
bb.build.exec_func() directly - instead we need to execute the tasks on
the server. To do this we use the buildFile command which already exists
for the purpose of supporting bitbake -b, and setVariable commands to
set up the appropriate datastore.
(I did look at passing the modified datastore to the buildFile command
instead of using setVar() on the main datastore, however its use of
databuilder makes that very difficult, and we'd also need a different
method of getting the changes in the datastore over to the worker as
well.)
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
scripts/lib/devtool/standard.py | 122 ++++++++++++++++++----------------------
scripts/lib/devtool/upgrade.py | 2 +-
2 files changed, 55 insertions(+), 69 deletions(-)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 06c508c..fbd8a71 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -378,7 +378,7 @@ def extract(args, config, basepath, workspace):
return 1
srctree = os.path.abspath(args.srctree)
- initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd)
+ initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil)
logger.info('Source tree extracted to %s' % srctree)
if initial_rev:
@@ -402,7 +402,7 @@ def sync(args, config, basepath, workspace):
return 1
srctree = os.path.abspath(args.srctree)
- initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd)
+ initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd, tinfoil)
logger.info('Source tree %s synchronized' % srctree)
if initial_rev:
@@ -412,35 +412,6 @@ def sync(args, config, basepath, workspace):
finally:
tinfoil.shutdown()
-class BbTaskExecutor(object):
- """Class for executing bitbake tasks for a recipe
-
- FIXME: This is very awkward. Unfortunately it's not currently easy to
- properly execute tasks outside of bitbake itself, until then this has to
- suffice if we are to handle e.g. linux-yocto's extra tasks
- """
-
- def __init__(self, rdata):
- self.rdata = rdata
- self.executed = []
-
- def exec_func(self, func, report):
- """Run bitbake task function"""
- if not func in self.executed:
- deps = self.rdata.getVarFlag(func, 'deps', False)
- if deps:
- for taskdepfunc in deps:
- self.exec_func(taskdepfunc, True)
- if report:
- logger.info('Executing %s...' % func)
- fn = self.rdata.getVar('FILE', True)
- localdata = bb.build._task_data(fn, func, self.rdata)
- try:
- bb.build.exec_func(func, localdata)
- except bb.build.FuncFailed as e:
- raise DevtoolError(str(e))
- self.executed.append(func)
-
def _prep_extract_operation(config, basepath, recipename, tinfoil=None):
"""HACK: Ugly workaround for making sure that requirements are met when
@@ -464,21 +435,10 @@ def _prep_extract_operation(config, basepath, recipename, tinfoil=None):
return tinfoil
-def _extract_source(srctree, keep_temp, devbranch, sync, d):
+def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil):
"""Extract sources of a recipe"""
- import bb.event
import oe.recipeutils
- def eventfilter(name, handler, event, d):
- """Bitbake event filter for devtool extract operation"""
- if name == 'base_eventhandler':
- return True
- else:
- return False
-
- if hasattr(bb.event, 'set_eventfilter'):
- bb.event.set_eventfilter(eventfilter)
-
pn = d.getVar('PN', True)
_check_compatible_recipe(pn, d)
@@ -504,19 +464,15 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
bb.utils.mkdirhier(srctree)
os.rmdir(srctree)
- # We don't want notes to be printed, they are too verbose
- origlevel = bb.logger.getEffectiveLevel()
- if logger.getEffectiveLevel() > logging.DEBUG:
- bb.logger.setLevel(logging.WARNING)
-
initial_rev = None
tempdir = tempfile.mkdtemp(prefix='devtool')
try:
+ tinfoil.logger.setLevel(logging.WARNING)
+
crd = d.createCopy()
# Make a subdir so we guard against WORKDIR==S
workdir = os.path.join(tempdir, 'workdir')
crd.setVar('WORKDIR', workdir)
- crd.setVar('T', os.path.join(tempdir, 'temp'))
if not crd.getVar('S', True).startswith(workdir):
# Usually a shared workdir recipe (kernel, gcc)
# Try to set a reasonable default
@@ -528,21 +484,55 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
# We don't want to move the source to STAGING_KERNEL_DIR here
crd.setVar('STAGING_KERNEL_DIR', '${S}')
- task_executor = BbTaskExecutor(crd)
+ is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d)
+ if not is_kernel_yocto:
+ crd.setVar('PATCHTOOL', 'git')
+ crd.setVar('PATCH_COMMIT_FUNCTIONS', '1')
+
+ # Apply our changes to the datastore to the server's datastore
+ for key in crd.localkeys():
+ tinfoil.config_data.setVar('%s_pn-%s' % (key, pn), crd.getVar(key, False))
+
+ tinfoil.config_data.setVar('STAMPS_DIR', os.path.join(tempdir, 'stamps'))
+ tinfoil.config_data.setVar('T', os.path.join(tempdir, 'temp'))
+ tinfoil.config_data.setVar('BUILDCFG_FUNCS', '')
+ tinfoil.config_data.setVar('BUILDCFG_HEADER', '')
+
+ tinfoil.set_event_mask(['bb.event.BuildStarted',
+ 'bb.event.BuildCompleted',
+ 'bb.event.TaskStarted',
+ 'logging.LogRecord',
+ 'bb.command.CommandCompleted',
+ 'bb.command.CommandFailed',
+ 'bb.build.TaskStarted',
+ 'bb.build.TaskSucceeded',
+ 'bb.build.TaskFailedSilent'])
+
+ def runtask(target, task):
+ if tinfoil.build_file(target, task):
+ while True:
+ event = tinfoil.wait_event(0.25)
+ if event:
+ if isinstance(event, bb.command.CommandCompleted):
+ break
+ elif isinstance(event, bb.command.CommandFailed):
+ raise DevtoolError('Task do_%s failed: %s' % (task, event.error))
+ elif isinstance(event, bb.build.TaskStarted):
+ logger.info('Executing %s...' % event._task)
+ elif isinstance(event, logging.LogRecord):
+ if event.levelno <= logging.INFO:
+ continue
+ logger.handle(event)
+
+ # we need virtual:native:/path/to/recipe if it's a BBCLASSEXTEND
+ fn = tinfoil.get_recipe_file(pn)
+ runtask(fn, 'unpack')
- crd.setVar('EXTERNALSRC_forcevariable', '')
-
- logger.info('Fetching %s...' % pn)
- task_executor.exec_func('do_fetch', False)
- logger.info('Unpacking...')
- task_executor.exec_func('do_unpack', False)
if bb.data.inherits_class('kernel-yocto', d):
# Extra step for kernel to populate the source directory
- logger.info('Doing kernel checkout...')
- task_executor.exec_func('do_kernel_checkout', False)
- srcsubdir = crd.getVar('S', True)
+ runtask(fn, 'kernel_checkout')
- task_executor.check_git = True
+ srcsubdir = crd.getVar('S', True)
# Move local source files into separate subdir
recipe_patches = [os.path.basename(patch) for patch in
@@ -572,7 +562,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
os.path.basename(fname) not in recipe_patches]
# Force separate S so that patch files can be left out from srctree
srcsubdir = tempfile.mkdtemp(dir=workdir)
- crd.setVar('S', srcsubdir)
+ tinfoil.config_data.setVar('S_task-patch', srcsubdir)
# Move source files to S
for path in src_files:
_move_file(os.path.join(workdir, path),
@@ -595,10 +585,8 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
(stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
initial_rev = stdout.rstrip()
- crd.setVar('PATCHTOOL', 'git')
-
logger.info('Patching...')
- task_executor.exec_func('do_patch', False)
+ runtask(fn, 'patch')
bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
@@ -606,7 +594,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
if bb.data.inherits_class('kernel-yocto', d):
# Store generate and store kernel config
logger.info('Generating kernel config')
- task_executor.exec_func('do_configure', False)
+ runtask(fn, 'configure')
kconfig = os.path.join(crd.getVar('B', True), '.config')
@@ -667,8 +655,6 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
shutil.copy2(kconfig, srctree)
finally:
- bb.logger.setLevel(origlevel)
-
if keep_temp:
logger.info('Preserving temporary directory %s' % tempdir)
else:
@@ -773,7 +759,7 @@ def modify(args, config, basepath, workspace):
initial_rev = None
commits = []
if not args.no_extract:
- initial_rev = _extract_source(srctree, False, args.branch, False, rd)
+ initial_rev = _extract_source(srctree, False, args.branch, False, rd, tinfoil)
if not initial_rev:
return 1
logger.info('Source tree extracted to %s' % srctree)
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 52f9ab1..d89e9a2 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -363,7 +363,7 @@ def upgrade(args, config, basepath, workspace):
rf = None
try:
- rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, rd)
+ rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, rd, tinfoil)
rev2, md5, sha256 = _extract_new_source(args.version, srctree, args.no_patch,
args.srcrev, args.branch, args.keep_temp,
tinfoil, rd)
--
2.5.5
^ permalink raw reply related
* [PATCH 08/14] recipetool: add OE lib path
From: Paul Eggleton @ 2016-12-13 7:09 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481612135.git.paul.eggleton@linux.intel.com>
The autotools code imports oe.package; we weren't experiencing a problem
with this probably due to OE itself adding that path previously.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
scripts/recipetool | 1 +
1 file changed, 1 insertion(+)
diff --git a/scripts/recipetool b/scripts/recipetool
index 1052cd2..882b702 100755
--- a/scripts/recipetool
+++ b/scripts/recipetool
@@ -73,6 +73,7 @@ def main():
logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
sys.exit(1)
logger.debug('Found bitbake path: %s' % bitbakepath)
+ scriptpath.add_oe_lib_path()
scriptutils.logger_setup_color(logger, global_args.color)
--
2.5.5
^ permalink raw reply related
* [PATCH 07/14] classes/patch: move several functions to oe.patch
From: Paul Eggleton @ 2016-12-13 7:09 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481612135.git.paul.eggleton@linux.intel.com>
Move patch_path(), src_patches() and should_apply() to oe.patch, making
them easier to call from elsewhere (particularly across the
UI/server boundary).
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
meta/classes/patch.bbclass | 105 ++------------------------------------------
meta/lib/oe/patch.py | 107 +++++++++++++++++++++++++++++++++++++++++++++
meta/lib/oe/recipeutils.py | 12 +++--
3 files changed, 116 insertions(+), 108 deletions(-)
diff --git a/meta/classes/patch.bbclass b/meta/classes/patch.bbclass
index 7ebae28..0e5b602 100644
--- a/meta/classes/patch.bbclass
+++ b/meta/classes/patch.bbclass
@@ -80,110 +80,13 @@ python patch_task_postfunc() {
}
def src_patches(d, all=False, expand=True):
- workdir = d.getVar('WORKDIR', True)
- fetch = bb.fetch2.Fetch([], d)
- patches = []
- sources = []
- for url in fetch.urls:
- local = patch_path(url, fetch, workdir, expand)
- if not local:
- if all:
- local = fetch.localpath(url)
- sources.append(local)
- continue
-
- urldata = fetch.ud[url]
- parm = urldata.parm
- patchname = parm.get('pname') or os.path.basename(local)
-
- apply, reason = should_apply(parm, d)
- if not apply:
- if reason:
- bb.note("Patch %s %s" % (patchname, reason))
- continue
-
- patchparm = {'patchname': patchname}
- if "striplevel" in parm:
- striplevel = parm["striplevel"]
- elif "pnum" in parm:
- #bb.msg.warn(None, "Deprecated usage of 'pnum' url parameter in '%s', please use 'striplevel'" % url)
- striplevel = parm["pnum"]
- else:
- striplevel = '1'
- patchparm['striplevel'] = striplevel
-
- patchdir = parm.get('patchdir')
- if patchdir:
- patchparm['patchdir'] = patchdir
-
- localurl = bb.fetch.encodeurl(('file', '', local, '', '', patchparm))
- patches.append(localurl)
-
- if all:
- return sources
-
- return patches
-
-def patch_path(url, fetch, workdir, expand=True):
- """Return the local path of a patch, or None if this isn't a patch"""
-
- local = fetch.localpath(url)
- base, ext = os.path.splitext(os.path.basename(local))
- if ext in ('.gz', '.bz2', '.Z'):
- if expand:
- local = os.path.join(workdir, base)
- ext = os.path.splitext(base)[1]
-
- urldata = fetch.ud[url]
- if "apply" in urldata.parm:
- apply = oe.types.boolean(urldata.parm["apply"])
- if not apply:
- return
- elif ext not in (".diff", ".patch"):
- return
-
- return local
+ import oe.patch
+ return oe.patch.src_patches(d, all, expand)
def should_apply(parm, d):
"""Determine if we should apply the given patch"""
-
- if "mindate" in parm or "maxdate" in parm:
- pn = d.getVar('PN', True)
- srcdate = d.getVar('SRCDATE_%s' % pn, True)
- if not srcdate:
- srcdate = d.getVar('SRCDATE', True)
-
- if srcdate == "now":
- srcdate = d.getVar('DATE', True)
-
- if "maxdate" in parm and parm["maxdate"] < srcdate:
- return False, 'is outdated'
-
- if "mindate" in parm and parm["mindate"] > srcdate:
- return False, 'is predated'
-
-
- if "minrev" in parm:
- srcrev = d.getVar('SRCREV', True)
- if srcrev and srcrev < parm["minrev"]:
- return False, 'applies to later revisions'
-
- if "maxrev" in parm:
- srcrev = d.getVar('SRCREV', True)
- if srcrev and srcrev > parm["maxrev"]:
- return False, 'applies to earlier revisions'
-
- if "rev" in parm:
- srcrev = d.getVar('SRCREV', True)
- if srcrev and parm["rev"] not in srcrev:
- return False, "doesn't apply to revision"
-
- if "notrev" in parm:
- srcrev = d.getVar('SRCREV', True)
- if srcrev and parm["notrev"] in srcrev:
- return False, "doesn't apply to revision"
-
- return True, None
+ import oe.patch
+ return oe.patch.should_apply(parm, d)
should_apply[vardepsexclude] = "DATE SRCDATE"
diff --git a/meta/lib/oe/patch.py b/meta/lib/oe/patch.py
index dbefd28..456ee70 100644
--- a/meta/lib/oe/patch.py
+++ b/meta/lib/oe/patch.py
@@ -769,3 +769,110 @@ class UserResolver(Resolver):
os.chdir(olddir)
raise
os.chdir(olddir)
+
+
+def patch_path(url, fetch, workdir, expand=True):
+ """Return the local path of a patch, or None if this isn't a patch"""
+
+ local = fetch.localpath(url)
+ base, ext = os.path.splitext(os.path.basename(local))
+ if ext in ('.gz', '.bz2', '.Z'):
+ if expand:
+ local = os.path.join(workdir, base)
+ ext = os.path.splitext(base)[1]
+
+ urldata = fetch.ud[url]
+ if "apply" in urldata.parm:
+ apply = oe.types.boolean(urldata.parm["apply"])
+ if not apply:
+ return
+ elif ext not in (".diff", ".patch"):
+ return
+
+ return local
+
+def src_patches(d, all=False, expand=True):
+ workdir = d.getVar('WORKDIR', True)
+ fetch = bb.fetch2.Fetch([], d)
+ patches = []
+ sources = []
+ for url in fetch.urls:
+ local = patch_path(url, fetch, workdir, expand)
+ if not local:
+ if all:
+ local = fetch.localpath(url)
+ sources.append(local)
+ continue
+
+ urldata = fetch.ud[url]
+ parm = urldata.parm
+ patchname = parm.get('pname') or os.path.basename(local)
+
+ apply, reason = should_apply(parm, d)
+ if not apply:
+ if reason:
+ bb.note("Patch %s %s" % (patchname, reason))
+ continue
+
+ patchparm = {'patchname': patchname}
+ if "striplevel" in parm:
+ striplevel = parm["striplevel"]
+ elif "pnum" in parm:
+ #bb.msg.warn(None, "Deprecated usage of 'pnum' url parameter in '%s', please use 'striplevel'" % url)
+ striplevel = parm["pnum"]
+ else:
+ striplevel = '1'
+ patchparm['striplevel'] = striplevel
+
+ patchdir = parm.get('patchdir')
+ if patchdir:
+ patchparm['patchdir'] = patchdir
+
+ localurl = bb.fetch.encodeurl(('file', '', local, '', '', patchparm))
+ patches.append(localurl)
+
+ if all:
+ return sources
+
+ return patches
+
+
+def should_apply(parm, d):
+ if "mindate" in parm or "maxdate" in parm:
+ pn = d.getVar('PN', True)
+ srcdate = d.getVar('SRCDATE_%s' % pn, True)
+ if not srcdate:
+ srcdate = d.getVar('SRCDATE', True)
+
+ if srcdate == "now":
+ srcdate = d.getVar('DATE', True)
+
+ if "maxdate" in parm and parm["maxdate"] < srcdate:
+ return False, 'is outdated'
+
+ if "mindate" in parm and parm["mindate"] > srcdate:
+ return False, 'is predated'
+
+
+ if "minrev" in parm:
+ srcrev = d.getVar('SRCREV', True)
+ if srcrev and srcrev < parm["minrev"]:
+ return False, 'applies to later revisions'
+
+ if "maxrev" in parm:
+ srcrev = d.getVar('SRCREV', True)
+ if srcrev and srcrev > parm["maxrev"]:
+ return False, 'applies to earlier revisions'
+
+ if "rev" in parm:
+ srcrev = d.getVar('SRCREV', True)
+ if srcrev and parm["rev"] not in srcrev:
+ return False, "doesn't apply to revision"
+
+ if "notrev" in parm:
+ srcrev = d.getVar('SRCREV', True)
+ if srcrev and parm["notrev"] in srcrev:
+ return False, "doesn't apply to revision"
+
+ return True, None
+
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index ae83aab..92fa431 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -382,6 +382,7 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True):
def get_recipe_local_files(d, patches=False, archives=False):
"""Get a list of local files in SRC_URI within a recipe."""
+ import oe.patch
uris = (d.getVar('SRC_URI', True) or "").split()
fetch = bb.fetch2.Fetch(uris, d)
# FIXME this list should be factored out somewhere else (such as the
@@ -393,7 +394,7 @@ def get_recipe_local_files(d, patches=False, archives=False):
for uri in uris:
if fetch.ud[uri].type == 'file':
if (not patches and
- bb.utils.exec_flat_python_func('patch_path', uri, fetch, '', expand=False)):
+ oe.patch.patch_path(uri, fetch, '', expand=False)):
continue
# Skip files that are referenced by absolute path
fname = fetch.ud[uri].basepath
@@ -418,10 +419,9 @@ def get_recipe_local_files(d, patches=False, archives=False):
def get_recipe_patches(d):
"""Get a list of the patches included in SRC_URI within a recipe."""
+ import oe.patch
+ patches = oe.patch.src_patches(d, expand=False)
patchfiles = []
- # Execute src_patches() defined in patch.bbclass - this works since that class
- # is inherited globally
- patches = bb.utils.exec_flat_python_func('src_patches', d, expand=False)
for patch in patches:
_, _, local, _, _, parm = bb.fetch.decodeurl(patch)
patchfiles.append(local)
@@ -438,9 +438,7 @@ def get_recipe_patched_files(d):
change mode ('A' for add, 'D' for delete or 'M' for modify)
"""
import oe.patch
- # Execute src_patches() defined in patch.bbclass - this works since that class
- # is inherited globally
- patches = bb.utils.exec_flat_python_func('src_patches', d, expand=False)
+ patches = oe.patch.src_patches(d, expand=False)
patchedfiles = {}
for patch in patches:
_, _, patchfile, _, _, parm = bb.fetch.decodeurl(patch)
--
2.5.5
^ permalink raw reply related
* [PATCH 06/14] classes/patch: move in logic to commit for additional tasks
From: Paul Eggleton @ 2016-12-13 7:09 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481612135.git.paul.eggleton@linux.intel.com>
If PATCHTOOL is "git", and PATCH_COMMIT_FUNCTIONS is set to "1", for
additional tasks between do_unpack and do_patch, make a git commit. This
logic was previously implemented in devtool itself, but it makes more
sense for it to be implemented in the patch class since that's where the
rest of the logic is for this (or in lib/oe/patch.py). It also makes
it possible for this to work with tinfoil2.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
meta/classes/patch.bbclass | 69 +++++++++++++++++++++++++++++++++++++++++
scripts/lib/devtool/standard.py | 37 +---------------------
2 files changed, 70 insertions(+), 36 deletions(-)
diff --git a/meta/classes/patch.bbclass b/meta/classes/patch.bbclass
index 2c1f58c..7ebae28 100644
--- a/meta/classes/patch.bbclass
+++ b/meta/classes/patch.bbclass
@@ -10,6 +10,75 @@ PATCH_GIT_USER_EMAIL ?= "oe.patch@oe"
inherit terminal
+python () {
+ if d.getVar('PATCHTOOL', True) == 'git' and d.getVar('PATCH_COMMIT_FUNCTIONS', True) == '1':
+ tasks = list(filter(lambda k: d.getVarFlag(k, "task", True), d.keys()))
+ extratasks = []
+ def follow_chain(task, endtask, chain=None):
+ if not chain:
+ chain = []
+ chain.append(task)
+ for othertask in tasks:
+ if othertask == task:
+ continue
+ if task == endtask:
+ for ctask in chain:
+ if ctask not in extratasks:
+ extratasks.append(ctask)
+ else:
+ deps = d.getVarFlag(othertask, 'deps', False)
+ if task in deps:
+ follow_chain(othertask, endtask, chain)
+ chain.pop()
+ follow_chain('do_unpack', 'do_patch')
+ try:
+ extratasks.remove('do_unpack')
+ except ValueError:
+ # For some recipes do_unpack doesn't exist, ignore it
+ pass
+
+ d.appendVarFlag('do_patch', 'prefuncs', ' patch_task_patch_prefunc')
+ for task in extratasks:
+ d.appendVarFlag(task, 'postfuncs', ' patch_task_postfunc')
+}
+
+python patch_task_patch_prefunc() {
+ # Prefunc for do_patch
+ func = d.getVar('BB_RUNTASK', True)
+ srcsubdir = d.getVar('S', True)
+
+ patchdir = os.path.join(srcsubdir, 'patches')
+ if os.path.exists(patchdir):
+ if os.listdir(patchdir):
+ d.setVar('PATCH_HAS_PATCHES_DIR', '1')
+ else:
+ os.rmdir(patchdir)
+}
+
+python patch_task_postfunc() {
+ # Prefunc for task functions between do_unpack and do_patch
+ import oe.patch
+ import shutil
+ func = d.getVar('BB_RUNTASK', True)
+ srcsubdir = d.getVar('S', True)
+
+ if os.path.exists(srcsubdir):
+ if func == 'do_patch':
+ haspatches = (d.getVar('PATCH_HAS_PATCHES_DIR', True) == '1')
+ patchdir = os.path.join(srcsubdir, 'patches')
+ if os.path.exists(patchdir):
+ shutil.rmtree(patchdir)
+ if haspatches:
+ stdout, _ = bb.process.run('git status --porcelain patches', cwd=srcsubdir)
+ if stdout:
+ bb.process.run('git checkout patches', cwd=srcsubdir)
+ stdout, _ = bb.process.run('git status --porcelain .', cwd=srcsubdir)
+ if stdout:
+ useroptions = []
+ oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d)
+ bb.process.run('git add .; git %s commit -a -m "Committing changes from %s\n\n%s"' % (' '.join(useroptions), func, oe.patch.GitApplyTree.ignore_commit_prefix + ' - from %s' % func), cwd=srcsubdir)
+}
+
def src_patches(d, all=False, expand=True):
workdir = d.getVar('WORKDIR', True)
fetch = bb.fetch2.Fetch([], d)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 87d3f5d..06c508c 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -442,41 +442,6 @@ class BbTaskExecutor(object):
self.executed.append(func)
-class PatchTaskExecutor(BbTaskExecutor):
- def __init__(self, rdata):
- import oe.patch
- self.check_git = False
- self.useroptions = []
- oe.patch.GitApplyTree.gitCommandUserOptions(self.useroptions, d=rdata)
- super(PatchTaskExecutor, self).__init__(rdata)
-
- def exec_func(self, func, report):
- from oe.patch import GitApplyTree
- srcsubdir = self.rdata.getVar('S', True)
- haspatches = False
- if func == 'do_patch':
- patchdir = os.path.join(srcsubdir, 'patches')
- if os.path.exists(patchdir):
- if os.listdir(patchdir):
- haspatches = True
- else:
- os.rmdir(patchdir)
-
- super(PatchTaskExecutor, self).exec_func(func, report)
- if self.check_git and os.path.exists(srcsubdir):
- if func == 'do_patch':
- if os.path.exists(patchdir):
- shutil.rmtree(patchdir)
- if haspatches:
- stdout, _ = bb.process.run('git status --porcelain patches', cwd=srcsubdir)
- if stdout:
- bb.process.run('git checkout patches', cwd=srcsubdir)
-
- stdout, _ = bb.process.run('git status --porcelain', cwd=srcsubdir)
- if stdout:
- bb.process.run('git add .; git %s commit -a -m "Committing changes from %s\n\n%s"' % (' '.join(self.useroptions), func, GitApplyTree.ignore_commit_prefix + ' - from %s' % func), cwd=srcsubdir)
-
-
def _prep_extract_operation(config, basepath, recipename, tinfoil=None):
"""HACK: Ugly workaround for making sure that requirements are met when
trying to extract a package. Returns the tinfoil instance to be used."""
@@ -563,7 +528,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
# We don't want to move the source to STAGING_KERNEL_DIR here
crd.setVar('STAGING_KERNEL_DIR', '${S}')
- task_executor = PatchTaskExecutor(crd)
+ task_executor = BbTaskExecutor(crd)
crd.setVar('EXTERNALSRC_forcevariable', '')
--
2.5.5
^ permalink raw reply related
* [PATCH 05/14] classes/base: fix license file checksumming when source not under TMPDIR
From: Paul Eggleton @ 2016-12-13 7:09 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481612135.git.paul.eggleton@linux.intel.com>
With the changes to the code for extracting source for a recipe, we are
properly executing the tasks for a recipe, which means their stamps (and
therefore signatures) are important. When running devtool extract on
the lsof recipe I noticed that do_fetch and do_unpack were executing a
second time when we called for do_patch, and this turned out to be
because LIC_FILES_CHKSUM in that recipe contains an entry which
is an absolute path (has ${S} at the start). Normally this wouldn't be
an issue since S is under TMPDIR and thus the existing code would ignore
it, however devtool's extraction code extracts to a temporary directory
which is not under TMPDIR; the result was the path to this file was not
being ignored and the second time around when the license file had been
extracted it was incorporated into the signature. We don't want this, so
explicitly exclude S as well as B and WORKDIR for good measure.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
meta/classes/base.bbclass | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 024fe43..19673e6 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -98,6 +98,9 @@ def get_lic_checksum_file_list(d):
filelist = []
lic_files = d.getVar("LIC_FILES_CHKSUM", True) or ''
tmpdir = d.getVar("TMPDIR", True)
+ s = d.getVar("S", True)
+ b = d.getVar("B", True)
+ workdir = d.getVar("WORKDIR", True)
urls = lic_files.split()
for url in urls:
@@ -109,7 +112,7 @@ def get_lic_checksum_file_list(d):
raise bb.fetch.MalformedUrl(url)
if path[0] == '/':
- if path.startswith(tmpdir):
+ if path.startswith((tmpdir, s, b, workdir)):
continue
filelist.append(path + ":" + str(os.path.exists(path)))
except bb.fetch.MalformedUrl:
--
2.5.5
^ permalink raw reply related
* [PATCH 04/14] devtool / recipetool: use tinfoil parsing API
From: Paul Eggleton @ 2016-12-13 7:09 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481612135.git.paul.eggleton@linux.intel.com>
Use Tinfoil.parse_recipe_file() and Tinfoil.parse_recipe() instead of
the recipeutils equivalents, and replace any local duplicate
implementations. This not only tidies up the code but also allows these
calls to work in memres mode.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
scripts/lib/devtool/__init__.py | 26 +++++++-------------------
scripts/lib/devtool/deploy.py | 2 +-
scripts/lib/devtool/standard.py | 2 +-
scripts/lib/devtool/upgrade.py | 2 +-
scripts/lib/recipetool/append.py | 21 ++++-----------------
scripts/lib/recipetool/newappend.py | 14 +-------------
scripts/lib/recipetool/setvar.py | 2 +-
7 files changed, 16 insertions(+), 53 deletions(-)
diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py
index 31ecb65..99c5534 100644
--- a/scripts/lib/devtool/__init__.py
+++ b/scripts/lib/devtool/__init__.py
@@ -120,34 +120,22 @@ def setup_tinfoil(config_only=False, basepath=None, tracking=False):
os.chdir(orig_cwd)
return tinfoil
-def get_recipe_file(cooker, pn):
- """Find recipe file corresponding a package name"""
- import oe.recipeutils
- recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
- if not recipefile:
- skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
- if skipreasons:
- logger.error('\n'.join(skipreasons))
- else:
- logger.error("Unable to find any recipe file matching %s" % pn)
- return recipefile
-
def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True):
- """Parse recipe of a package"""
- import oe.recipeutils
- recipefile = get_recipe_file(tinfoil.cooker, pn)
- if not recipefile:
- # Error already logged
+ """Parse the specified recipe"""
+ try:
+ recipefile = tinfoil.get_recipe_file(pn)
+ except bb.providers.NoProvider as e:
+ logger.error(str(e))
return None
if appends:
- append_files = tinfoil.cooker.collection.get_file_appends(recipefile)
+ append_files = tinfoil.get_file_appends(recipefile)
if filter_workspace:
# Filter out appends from the workspace
append_files = [path for path in append_files if
not path.startswith(config.workspace_path)]
else:
append_files = None
- return oe.recipeutils.parse_recipe(tinfoil.cooker, recipefile, append_files)
+ return tinfoil.parse_recipe_file(recipefile, appends, append_files)
def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False):
"""
diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py
index c4c7bf6..db7dffa 100644
--- a/scripts/lib/devtool/deploy.py
+++ b/scripts/lib/devtool/deploy.py
@@ -156,7 +156,7 @@ def deploy(args, config, basepath, workspace):
tinfoil = setup_tinfoil(basepath=basepath)
try:
try:
- rd = oe.recipeutils.parse_recipe_simple(tinfoil.cooker, args.recipename, tinfoil.config_data)
+ rd = tinfoil.parse_recipe(args.recipename)
except Exception as e:
raise DevtoolError('Exception parsing recipe %s: %s' %
(args.recipename, e))
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index e4d2a57..87d3f5d 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -224,7 +224,7 @@ def add(args, config, basepath, workspace):
tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
try:
- rd = oe.recipeutils.parse_recipe(tinfoil.cooker, recipefile, None)
+ rd = tinfoil.parse_recipe_file(recipefile, False)
if not rd:
return 1
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index a4239f1..52f9ab1 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -320,7 +320,7 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, workspace, tinfoil
newvalues['SRC_URI[md5sum]'] = md5
newvalues['SRC_URI[sha256sum]'] = sha256
- rd = oe.recipeutils.parse_recipe(tinfoil.cooker, fullpath, None)
+ rd = tinfoil.parse_recipe_file(fullpath, False)
oe.recipeutils.patch_recipe(rd, fullpath, newvalues)
return fullpath, copied
diff --git a/scripts/lib/recipetool/append.py b/scripts/lib/recipetool/append.py
index 1e0fc1e..3e85a0c 100644
--- a/scripts/lib/recipetool/append.py
+++ b/scripts/lib/recipetool/append.py
@@ -97,25 +97,12 @@ def find_target_file(targetpath, d, pkglist=None):
recipes[targetpath].append('!%s' % pn)
return recipes
-def _get_recipe_file(cooker, pn):
- import oe.recipeutils
- recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
- if not recipefile:
- skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
- if skipreasons:
- logger.error('\n'.join(skipreasons))
- else:
- logger.error("Unable to find any recipe file matching %s" % pn)
- return recipefile
-
def _parse_recipe(pn, tinfoil):
- import oe.recipeutils
- recipefile = _get_recipe_file(tinfoil.cooker, pn)
- if not recipefile:
- # Error already logged
+ try:
+ rd = tinfoil.parse_recipe(pn)
+ except bb.providers.NoProvider as e:
+ logger.error(str(e))
return None
- append_files = tinfoil.cooker.collection.get_file_appends(recipefile)
- rd = oe.recipeutils.parse_recipe(tinfoil.cooker, recipefile, append_files)
return rd
def determine_file_source(targetpath, rd):
diff --git a/scripts/lib/recipetool/newappend.py b/scripts/lib/recipetool/newappend.py
index fbdd7bc..3760840 100644
--- a/scripts/lib/recipetool/newappend.py
+++ b/scripts/lib/recipetool/newappend.py
@@ -39,18 +39,6 @@ def tinfoil_init(instance):
tinfoil = instance
-def _get_recipe_file(cooker, pn):
- import oe.recipeutils
- recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
- if not recipefile:
- skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
- if skipreasons:
- logger.error('\n'.join(skipreasons))
- else:
- logger.error("Unable to find any recipe file matching %s" % pn)
- return recipefile
-
-
def layer(layerpath):
if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')):
raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath))
@@ -60,7 +48,7 @@ def layer(layerpath):
def newappend(args):
import oe.recipeutils
- recipe_path = _get_recipe_file(tinfoil.cooker, args.target)
+ recipe_path = tinfoil.get_recipe_file(args.target)
rd = tinfoil.config_data.createCopy()
rd.setVar('FILE', recipe_path)
diff --git a/scripts/lib/recipetool/setvar.py b/scripts/lib/recipetool/setvar.py
index 85701c0..9de315a 100644
--- a/scripts/lib/recipetool/setvar.py
+++ b/scripts/lib/recipetool/setvar.py
@@ -51,7 +51,7 @@ def setvar(args):
if args.recipe_only:
patches = [oe.recipeutils.patch_recipe_file(args.recipefile, varvalues, patch=args.patch)]
else:
- rd = oe.recipeutils.parse_recipe(tinfoil.cooker, args.recipefile, None)
+ rd = tinfoil.parse_recipe_file(args.recipefile, False)
if not rd:
return 1
patches = oe.recipeutils.patch_recipe(rd, args.recipefile, varvalues, patch=args.patch)
--
2.5.5
^ permalink raw reply related
* [PATCH 03/14] oe-selftest: use tinfoil.parse_recipe()
From: Paul Eggleton @ 2016-12-13 7:09 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481612135.git.paul.eggleton@linux.intel.com>
Use tinfoil.parse_recipe() in order to allow oe-selftest to be used in
memres mode.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
meta/lib/oeqa/utils/commands.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index 47cb6b7..6e05995 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -221,9 +221,7 @@ def runqemu(pn, ssh=True):
import oeqa.targetcontrol
tinfoil.config_data.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage")
tinfoil.config_data.setVar("TEST_QEMUBOOT_TIMEOUT", "1000")
- import oe.recipeutils
- recipefile = oe.recipeutils.pn_to_recipe(tinfoil.cooker, pn)
- recipedata = oe.recipeutils.parse_recipe(tinfoil.cooker, recipefile, [])
+ recipedata = tinfoil.parse_recipe(pn)
# The QemuRunner log is saved out, but we need to ensure it is at the right
# log level (and then ensure that since it's a child of the BitBake logger,
--
2.5.5
^ permalink raw reply related
* [PATCH 02/14] oe-selftest: make tinfoil quiet when using to start QEMU
From: Paul Eggleton @ 2016-12-13 7:09 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481612135.git.paul.eggleton@linux.intel.com>
We don't need to see the parsing/cache loading message in the
oe-selftest output, so use the newly added quiet option to disable it.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
meta/lib/oeqa/utils/commands.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index 5cd0f74..47cb6b7 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -215,7 +215,7 @@ def runqemu(pn, ssh=True):
import bb.build
tinfoil = bb.tinfoil.Tinfoil()
- tinfoil.prepare(False)
+ tinfoil.prepare(config_only=False, quiet=True)
try:
tinfoil.logger.setLevel(logging.WARNING)
import oeqa.targetcontrol
--
2.5.5
^ permalink raw reply related
* [PATCH 01/14] lib/oe/recipeutils: use cooker function instead of bb.providers
From: Paul Eggleton @ 2016-12-13 7:09 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481612135.git.paul.eggleton@linux.intel.com>
We now have a function in cooker itself that can do this lookup;
additionally, the rewritten tinfoil's cooker adapter has its own
implementation that can work remotely, so if we use it then this
function can work in that scenario as well.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
meta/lib/oe/recipeutils.py | 13 ++-----------
1 file changed, 2 insertions(+), 11 deletions(-)
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index 1589feb..ae83aab 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -29,18 +29,9 @@ meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
def pn_to_recipe(cooker, pn, mc=''):
"""Convert a recipe name (PN) to the path to the recipe file"""
- import bb.providers
- if pn in cooker.recipecaches[mc].pkg_pn:
- best = bb.providers.findBestProvider(pn, cooker.data, cooker.recipecaches[mc], cooker.recipecaches[mc].pkg_pn)
- return best[3]
- elif pn in cooker.recipecaches[mc].providers:
- filenames = cooker.recipecaches[mc].providers[pn]
- eligible, foundUnique = bb.providers.filterProviders(filenames, pn, cooker.expanded_data, cooker.recipecaches[mc])
- filename = eligible[0]
- return filename
- else:
- return None
+ best = cooker.findBestProvider(pn, mc)
+ return best[3]
def get_unavailable_reasons(cooker, pn):
--
2.5.5
^ permalink raw reply related
* [PATCH 00/14] Tinfoil rework - OE-Core changes
From: Paul Eggleton @ 2016-12-13 7:09 UTC (permalink / raw)
To: openembedded-core
Tinfoil is an API that allows you to write simple utilities that call into
BitBake code to read variables, parse recipes, etc. (as used by various
scripts in BitBake & OE, including bitbake-layers, devtool, etc.) This
patchset reworks tinfoil in order to address several limitations of the
current implementation, first and foremost that it doesn't currently work
in bitbake's memory resident mode. This rework is also known as "tinfoil2"
although the classes and modules have not changed name. This patchset
depends upon the corresponding patchset sent to the bitbake-devel list and
both sets should be applied together in order for things to remain working.
Note that whilst these patchsets make some significant improvements to how
the system behaves in memory resident mode, there are still a number of
issues that prevent memory resident mode from being reliable. See this
wiki page for a few more details:
https://wiki.yoctoproject.org/wiki/index.php?title=Tinfoil2
I will be converting some of these into bugzilla entries after this
patchset and the corresponding one for bitbake get merged.
The following changes since commit d62f18c39bc0ed3b0f5ac8465b393c15f2143ecf:
targetloader.py: drop test for ClassType (2016-12-12 15:16:39 +0000)
are available in the git repository at:
git://git.openembedded.org/openembedded-core-contrib paule/tinfoil2-oecore
http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/tinfoil2-oecore
Paul Eggleton (14):
lib/oe/recipeutils: use cooker function instead of bb.providers
oe-selftest: make tinfoil quiet when using to start QEMU
oe-selftest: use tinfoil.parse_recipe()
devtool / recipetool: use tinfoil parsing API
classes/base: fix license file checksumming when source not under
TMPDIR
classes/patch: move in logic to commit for additional tasks
classes/patch: move several functions to oe.patch
recipetool: add OE lib path
devtool: fix extraction of source to work in memres mode
lib/oe/recipeutils: drop parse_recipe_simple()
devtool: extract: disable basehash mismatch errors
devtool: prevent BBHandledException from showing traceback
oe-selftest: devtool: improve test_devtool_modify slightly
oe-selftest: add basic tinfoil tests
meta/classes/base.bbclass | 5 +-
meta/classes/patch.bbclass | 168 +++++++++++++++---------------------
meta/lib/oe/patch.py | 107 +++++++++++++++++++++++
meta/lib/oe/recipeutils.py | 49 ++---------
meta/lib/oeqa/selftest/devtool.py | 15 +++-
meta/lib/oeqa/selftest/tinfoil.py | 146 +++++++++++++++++++++++++++++++
meta/lib/oeqa/utils/commands.py | 6 +-
scripts/devtool | 11 ++-
scripts/lib/devtool/__init__.py | 26 ++----
scripts/lib/devtool/deploy.py | 2 +-
scripts/lib/devtool/standard.py | 160 ++++++++++++----------------------
scripts/lib/devtool/upgrade.py | 4 +-
scripts/lib/recipetool/append.py | 21 +----
scripts/lib/recipetool/newappend.py | 14 +--
scripts/lib/recipetool/setvar.py | 2 +-
scripts/recipetool | 1 +
16 files changed, 429 insertions(+), 308 deletions(-)
create mode 100644 meta/lib/oeqa/selftest/tinfoil.py
--
2.5.5
^ permalink raw reply
* Re: [PATCH 2/8] oeqa/sdkext/devtool.py: remove workspace/sources before running test cases
From: Robert Yang @ 2016-12-13 6:47 UTC (permalink / raw)
To: Paul Eggleton
Cc: Francisco Pedraza, Limon, Anibal, Lopez, Mariano,
openembedded-core
In-Reply-To: <7884104.xbYfD5GGaH@peggleto-mobl.ger.corp.intel.com>
On 12/13/2016 02:29 PM, Paul Eggleton wrote:
> On Tue, 13 Dec 2016 13:56:05 Robert Yang wrote:
>> Hi Paul,
>>
>> Thanks for reply, please see my comments inline.
>>
>> On 12/13/2016 12:45 PM, Paul Eggleton wrote:
>>> On Wed, 16 Nov 2016 22:19:31 Robert Yang wrote:
>>>> Fixed:
>>>> MACHINE = "qemux86-64"
>>>> require conf/multilib.conf
>>>> MULTILIBS = "multilib:lib32"
>>>> DEFAULTTUNE_virtclass-multilib-lib32 = "x86"
>>>>
>>>> $ bitbake core-image-minimal -cpopulate_sdk_ext
>>>> [snip]
>>>> ERROR: Source tree path
>>>> /path/to/tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/testsdk
>>>> ex
>>>> t/tc/workspace/sources/v4l2loopback-driver already exists and is not
>>>> empty\n' [snip]
>>>>
>>>> This is because the test case will run twice
>>>> (environment-setup-core2-64-poky-linux and
>>>> environment-setup-x86-pokymllib32-linux), it would fail in the second
>>>> run, 'devtool reset' can not remove sources, so remove it before running
>>>> test cases.
>>>>
>>>> [YOCTO #10647]
>>>>
>>>> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
>>>> ---
>>>>
>>>> meta/lib/oeqa/sdkext/devtool.py | 3 +++
>>>> 1 file changed, 3 insertions(+)
>>>>
>>>> diff --git a/meta/lib/oeqa/sdkext/devtool.py
>>>> b/meta/lib/oeqa/sdkext/devtool.py index 65f41f6..f101eb6 100644
>>>> --- a/meta/lib/oeqa/sdkext/devtool.py
>>>> +++ b/meta/lib/oeqa/sdkext/devtool.py
>>>>
>>>> @@ -15,6 +15,9 @@ class DevtoolTest(oeSDKExtTest):
>>>> self.myapp_cmake_dst = os.path.join(self.tc.sdktestdir,
>>>>
>>>> "myapp_cmake") shutil.copytree(self.myapp_cmake_src,
>>>> self.myapp_cmake_dst)
>>>>
>>>> + # Clean sources dir to make "git clone" can run again
>>>> + shutil.rmtree(os.path.join(self.tc.sdktestdir,
>>>> "tc/workspace/sources"), True) +
>>>>
>>>> def _test_devtool_build(self, directory):
>>>> self._run('devtool add myapp %s' % directory)
>>>
>>>> try:
>>> It seems to me that's what's missing here is a proper teardown process
>>> like we have for oe-selftest, so that tests clean up after themselves
>>> whether they succeed or fail. I'm unsure as to whether that is part of
>>> the plan for the new QA refactoring though.
>>
>> There is already a 'devtool reset' which can do the cleanup, but it
>> can't remove sources as I said in the commit log.
>
> devtool reset not deleting the source tree is intentional - I don't want
Add an option like "devtool reset --force" ? When I met the error, the first
I did was check "devtool reset", but there is no way to remove resources, so I
have to remove it here to allow multilib test cases runs. Or maybe fix
"devtool add" to check the git repo correclty ? For example, when there
is already a repo, just update it rather than clone it again.
The multilib + testsdkext would fail without this fix.
// Robert
> people accidentally losing changes to their source. In any case though it's
> not about what we do here but where we do it.
>
> Cheers,
> Paul
>
^ permalink raw reply
* Re: [PATCH 5/8] oeqa/sdkext/devtool.py: don't reset when the test is failed
From: Robert Yang @ 2016-12-13 6:39 UTC (permalink / raw)
To: Paul Eggleton; +Cc: openembedded-core
In-Reply-To: <2244665.P00pJDD7kM@peggleto-mobl.ger.corp.intel.com>
On 12/13/2016 02:33 PM, Paul Eggleton wrote:
> On Tue, 13 Dec 2016 14:01:08 Robert Yang wrote:
>> On 12/13/2016 12:48 PM, Paul Eggleton wrote:
>>> On Wed, 16 Nov 2016 22:19:34 Robert Yang wrote:
>>>> The contents are helpful to debug when the error happens.
>>>
>>> In this case removing this is OK because we're operating on an eSDK that
>>> the tests install. However, this wouldn't be appropriate for the devtool
>>> oe- selftest tests since they are operating on the user's build system
>>> itself - not that you probably have any intention of changing those, I
>>> just wanted to note that.
>>
>> Sorry, I don't quite understand, does selftest uses oeqa/sdkext/devtool.py,
>> please ?
>
> No, this is for bitbake -c testsdkext.
>
>> Even if it uses devtool.py, I still think that we need keep the
>> error contents for debugging when the error happens. It's very hard to
>> debug when there is no error log or no workspace.
>
> I mean the equivalent oe-selftest tests in oeqa/selftest/devtool.py.
>
> Actually, I've realised something that applies here as well - leaving the
> files around would be OK if we stopped on the first failure, but we don't -
> the next test proceeds after it. How will you be able to rely on what's in the
> workspace if several other tests have run afterwards - not to mention ensure
> those tests aren't disrupted by the leftover files?
That's a problem, but we really need consider debugging, otherwise it's
painful when test are failed but nothing left. How about run test cases
in different workspaces ?
// Robert
>
> Cheers,
> Paul
>
^ permalink raw reply
* [PATCH] libxi: 1.7.7 -> 1.7.8
From: Huang Qiyu @ 2016-12-13 14:36 UTC (permalink / raw)
To: openembedded-core
Upgrade libxi from 1.7.7 to 1.7.8.
Signed-off-by: Huang Qiyu <huangqy.fnst@cn.fujitsu.com>
---
meta/recipes-graphics/xorg-lib/{libxi_1.7.7.bb => libxi_1.7.8.bb} | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
rename meta/recipes-graphics/xorg-lib/{libxi_1.7.7.bb => libxi_1.7.8.bb} (81%)
diff --git a/meta/recipes-graphics/xorg-lib/libxi_1.7.7.bb b/meta/recipes-graphics/xorg-lib/libxi_1.7.8.bb
similarity index 81%
rename from meta/recipes-graphics/xorg-lib/libxi_1.7.7.bb
rename to meta/recipes-graphics/xorg-lib/libxi_1.7.8.bb
index 934de08..cc3e09e 100644
--- a/meta/recipes-graphics/xorg-lib/libxi_1.7.7.bb
+++ b/meta/recipes-graphics/xorg-lib/libxi_1.7.8.bb
@@ -17,6 +17,6 @@ PE = "1"
XORG_PN = "libXi"
-SRC_URI[md5sum] = "cc0883a898222d50ff79af3f83595823"
-SRC_URI[sha256sum] = "996f834fa57b9b33ba36690f6f5c6a29320bc8213022943912462d8015b1e030"
+SRC_URI[md5sum] = "94afc83e553d3c38a153f8f60301fd62"
+SRC_URI[sha256sum] = "d8f2fa8d53141c41ff521627df9b2fa9c05f6f142fd9881152bab36549ac27bb"
--
2.7.4
^ permalink raw reply related
* Re: [PATCH 5/8] oeqa/sdkext/devtool.py: don't reset when the test is failed
From: Paul Eggleton @ 2016-12-13 6:33 UTC (permalink / raw)
To: Robert Yang; +Cc: openembedded-core
In-Reply-To: <bcfd077b-eee2-fa8e-dfd3-a2470d07ec3f@windriver.com>
On Tue, 13 Dec 2016 14:01:08 Robert Yang wrote:
> On 12/13/2016 12:48 PM, Paul Eggleton wrote:
> > On Wed, 16 Nov 2016 22:19:34 Robert Yang wrote:
> >> The contents are helpful to debug when the error happens.
> >
> > In this case removing this is OK because we're operating on an eSDK that
> > the tests install. However, this wouldn't be appropriate for the devtool
> > oe- selftest tests since they are operating on the user's build system
> > itself - not that you probably have any intention of changing those, I
> > just wanted to note that.
>
> Sorry, I don't quite understand, does selftest uses oeqa/sdkext/devtool.py,
> please ?
No, this is for bitbake -c testsdkext.
> Even if it uses devtool.py, I still think that we need keep the
> error contents for debugging when the error happens. It's very hard to
> debug when there is no error log or no workspace.
I mean the equivalent oe-selftest tests in oeqa/selftest/devtool.py.
Actually, I've realised something that applies here as well - leaving the
files around would be OK if we stopped on the first failure, but we don't -
the next test proceeds after it. How will you be able to rely on what's in the
workspace if several other tests have run afterwards - not to mention ensure
those tests aren't disrupted by the leftover files?
Cheers,
Paul
--
Paul Eggleton
Intel Open Source Technology Centre
^ permalink raw reply
* Re: [PATCH 2/8] oeqa/sdkext/devtool.py: remove workspace/sources before running test cases
From: Paul Eggleton @ 2016-12-13 6:29 UTC (permalink / raw)
To: Robert Yang
Cc: Francisco Pedraza, Limon, Anibal, Lopez, Mariano,
openembedded-core
In-Reply-To: <10c5001f-1209-acd7-dc7a-7c64792b36b2@windriver.com>
On Tue, 13 Dec 2016 13:56:05 Robert Yang wrote:
> Hi Paul,
>
> Thanks for reply, please see my comments inline.
>
> On 12/13/2016 12:45 PM, Paul Eggleton wrote:
> > On Wed, 16 Nov 2016 22:19:31 Robert Yang wrote:
> >> Fixed:
> >> MACHINE = "qemux86-64"
> >> require conf/multilib.conf
> >> MULTILIBS = "multilib:lib32"
> >> DEFAULTTUNE_virtclass-multilib-lib32 = "x86"
> >>
> >> $ bitbake core-image-minimal -cpopulate_sdk_ext
> >> [snip]
> >> ERROR: Source tree path
> >> /path/to/tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/testsdk
> >> ex
> >> t/tc/workspace/sources/v4l2loopback-driver already exists and is not
> >> empty\n' [snip]
> >>
> >> This is because the test case will run twice
> >> (environment-setup-core2-64-poky-linux and
> >> environment-setup-x86-pokymllib32-linux), it would fail in the second
> >> run, 'devtool reset' can not remove sources, so remove it before running
> >> test cases.
> >>
> >> [YOCTO #10647]
> >>
> >> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
> >> ---
> >>
> >> meta/lib/oeqa/sdkext/devtool.py | 3 +++
> >> 1 file changed, 3 insertions(+)
> >>
> >> diff --git a/meta/lib/oeqa/sdkext/devtool.py
> >> b/meta/lib/oeqa/sdkext/devtool.py index 65f41f6..f101eb6 100644
> >> --- a/meta/lib/oeqa/sdkext/devtool.py
> >> +++ b/meta/lib/oeqa/sdkext/devtool.py
> >>
> >> @@ -15,6 +15,9 @@ class DevtoolTest(oeSDKExtTest):
> >> self.myapp_cmake_dst = os.path.join(self.tc.sdktestdir,
> >>
> >> "myapp_cmake") shutil.copytree(self.myapp_cmake_src,
> >> self.myapp_cmake_dst)
> >>
> >> + # Clean sources dir to make "git clone" can run again
> >> + shutil.rmtree(os.path.join(self.tc.sdktestdir,
> >> "tc/workspace/sources"), True) +
> >>
> >> def _test_devtool_build(self, directory):
> >> self._run('devtool add myapp %s' % directory)
> >
> >> try:
> > It seems to me that's what's missing here is a proper teardown process
> > like we have for oe-selftest, so that tests clean up after themselves
> > whether they succeed or fail. I'm unsure as to whether that is part of
> > the plan for the new QA refactoring though.
>
> There is already a 'devtool reset' which can do the cleanup, but it
> can't remove sources as I said in the commit log.
devtool reset not deleting the source tree is intentional - I don't want
people accidentally losing changes to their source. In any case though it's
not about what we do here but where we do it.
Cheers,
Paul
--
Paul Eggleton
Intel Open Source Technology Centre
^ permalink raw reply
* Re: [PATCH 4/8] oe-publish-sdk: add pyshtables.py to .gitignore
From: Robert Yang @ 2016-12-13 6:03 UTC (permalink / raw)
To: Paul Eggleton; +Cc: openembedded-core
In-Reply-To: <19498001.0ZzLqTTt5f@peggleto-mobl.ger.corp.intel.com>
On 12/13/2016 12:59 PM, Paul Eggleton wrote:
> On Wed, 16 Nov 2016 22:19:33 Robert Yang wrote:
>> Fixed:
>> MACHINE = "qemux86-64"
>> require conf/multilib.conf
>> MULTILIBS = "multilib:lib32"
>> DEFAULTTUNE_virtclass-multilib-lib32 = "x86"
>>
>> $ bitbake core-image-minimal -cpopulate_sdk_ext
>> [snip]
>> ERROR: Failed to update metadata as there have been changes made to it.
>> Aborting.\nERROR: Changed files:\nb' M
>> poky/bitbake/lib/bb/pysh/pyshtables.py\\n'\n" [snip]
>>
>> This is because the test case will run twice
>> (environment-setup-core2-64-poky-linux and
>> environment-setup-x86-pokymllib32-linux), it would fail in the second
>> run since pyshtables.py is regenerated in the first run. This file is
>> generated automatically, publish it doesn't make any sense, so add it to
>> .gitignore.
>>
>> [YOCTO #10647]
>
> The actual fix looks OK but I don't think 10647 is really the right bug
> number. Coincidentally today someone else reported this issue, so this would
> probably be a better one:
>
> https://bugzilla.yoctoproject.org/show_bug.cgi?id=10796
Thanks, I will update it.
// Robert
>
>> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
>> ---
>> scripts/oe-publish-sdk | 2 +-
>> 1 file changed, 1 insertion(+), 1 deletion(-)
>>
>> diff --git a/scripts/oe-publish-sdk b/scripts/oe-publish-sdk
>> index d95c623..e2b1b95 100755
>> --- a/scripts/oe-publish-sdk
>> +++ b/scripts/oe-publish-sdk
>> @@ -116,7 +116,7 @@ def publish(args):
>> cmd_common = "if [ ! -e .git ]; then"
>> cmd_common += " git init .;"
>> cmd_common += " mv .git/hooks/post-update.sample
>> .git/hooks/post-update;" - cmd_common += " echo '*.pyc\n*.pyo' >
>> .gitignore;"
>> + cmd_common += " echo '*.pyc\n*.pyo\npyshtables.py' > .gitignore;"
>> cmd_common += "fi;"
>> cmd_common += "git add -A .;"
>> cmd_common += "git config user.email 'oe@oe.oe' && git config user.name
>> 'OE' && git commit -q -m 'init repo' || true;"
>
> Cheers,
> Paul
>
^ permalink raw reply
* Re: [PATCH 1/8] populate_sdk_ext.bbclass: install multilib targets as populate_sdk does
From: Robert Yang @ 2016-12-13 6:03 UTC (permalink / raw)
To: Paul Eggleton; +Cc: openembedded-core
In-Reply-To: <7337875.m9BQhVBZV7@peggleto-mobl.ger.corp.intel.com>
On 12/13/2016 12:55 PM, Paul Eggleton wrote:
> Hi Robert,
>
> There are a bunch of changes in here that don't relate to the multilib fix -
> details below.
Thanks, I will split them into two commits.
>
> On Wed, 16 Nov 2016 22:19:30 Robert Yang wrote:
>> Fixed:
>> MACHINE = "qemux86-64"
>> require conf/multilib.conf
>> MULTILIBS = "multilib:lib32"
>> DEFAULTTUNE_virtclass-multilib-lib32 = "x86"
>>
>> $ bitbake core-image-minimal -cpopulate_sdk_ext
>> [snip]
>> Testing
>> /buildarea/lyang1/test_po/tmp/work/qemux86_64-poky-linux/core-image-minimal
>> /1.0-r0/testsdkext//tc/environment-setup-x86-pokymllib32-linux test_cvs
>> (oeqa.sdk.buildcvs.BuildCvsTest) ... FAIL
>> [snip]
>>
>> It was failed because no lib32 toolchains.
>>
>> The fixes include:
>> * Set SDK_TARGETS correctly
>> * Return multilib depends in get_ext_sdk_depends()
>> * Write information to all environment-setup-* scripts.
>>
>> [YOCTO #10647]
>>
>> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
>> ---
>> meta/classes/populate_sdk_ext.bbclass | 61
>> ++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 23
>> deletions(-)
>>
>> diff --git a/meta/classes/populate_sdk_ext.bbclass
>> b/meta/classes/populate_sdk_ext.bbclass index 26b5ca6..ce9c40a 100644
>> --- a/meta/classes/populate_sdk_ext.bbclass
>> +++ b/meta/classes/populate_sdk_ext.bbclass
>> @@ -39,7 +39,7 @@ SDK_LOCAL_CONF_BLACKLIST ?= "CONF_VERSION \
>> SDK_INHERIT_BLACKLIST ?= "buildhistory icecc"
>> SDK_UPDATE_URL ?= ""
>>
>> -SDK_TARGETS ?= "${PN}"
>> +SDK_TARGETS ?= "${@multilib_pkg_extend(d, d.getVar('BPN', True))}"
>>
>> def get_sdk_install_targets(d, images_only=False):
>> sdk_install_targets = ''
>> @@ -562,38 +562,52 @@ SDK_PRE_INSTALL_COMMAND_task-populate-sdk-ext =
>> "${sdk_ext_preinst}" sdk_ext_postinst() {
>> printf "\nExtracting buildtools...\n"
>> cd $target_sdk_dir
>> - env_setup_script="$target_sdk_dir/environment-
> setup-${REAL_MULTIMACH_TARGE
>> T_SYS}" - printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} >
>> buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat
>> buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and
>> needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
>> + env_setup_scripts="`ls $target_sdk_dir/environment-setup-*`"
>> + ./${SDK_BUILDTOOLS_INSTALLER} -d buildtools -y > buildtools.log
>> + if [ $? -ne 0 ]; then
>> + echo 'ERROR: buildtools installation failed:'
>> + cat buildtools.log
>> + for e in $env_setup_scripts; do
>> + echo "echo 'ERROR: this SDK was not fully installed and needs
>> reinstalling'" >> $e + done
>> + exit 1
>> + fi
>>
>
> This change isn't entirely related to the multilib changes.
>
>> # Delete the buildtools tar file since it won't be used again
>> rm -f ./${SDK_BUILDTOOLS_INSTALLER}
>> # We don't need the log either since it succeeded
>> rm -f buildtools.log
>>
>> - # Make sure when the user sets up the environment, they also get
>> - # the buildtools-tarball tools in their path.
>> - echo ". $target_sdk_dir/buildtools/environment-setup*" >>
>> $env_setup_script -
>> - # Allow bitbake environment setup to be ran as part of this sdk.
>> - echo "export OE_SKIP_SDK_CHECK=1" >> $env_setup_script
>> + for e in $env_setup_scripts; do
>> + # Make sure when the user sets up the environment, they also get
>> + # the buildtools-tarball tools in their path.
>> + echo ". $target_sdk_dir/buildtools/environment-setup*" >> $e
>>
>> - # A bit of another hack, but we need this in the path only for devtool
>> - # so put it at the end of $PATH.
>> - echo "export
>> PATH=$target_sdk_dir/sysroots/${SDK_SYS}${bindir_nativesdk}:\$PATH" >>
>> $env_setup_script + # Allow bitbake environment setup to be ran as part
> of
>> this sdk. + echo "export OE_SKIP_SDK_CHECK=1" >> $e
>>
>> - echo "printf 'SDK environment now set up; additionally you may now run
>> devtool to perform development tasks.\nRun devtool --help for further
>> details.\n'" >> $env_setup_script -
>> - # Warn if trying to use external bitbake and the ext SDK together
>> - echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING: attempting to use
>> the extensible SDK in an environment set up to run bitbake - this may lead
>> to unexpected results. Please source this script in a new shell session
>> instead.') || true" >> $env_setup_script + # A bit of another hack, but
> we
>> need this in the path only for devtool + # so put it at the end of
> $PATH.
>> + echo "export
>> PATH=$target_sdk_dir/sysroots/${SDK_SYS}${bindir_nativesdk}:\$PATH" >> $e
>> + echo "printf 'SDK environment now set up; additionally you may now
> run
>> devtool to perform development tasks.\nRun devtool --help for further
>> details.\n'" >> $e + # Warn if trying to use external bitbake and the
> ext
>> SDK together + echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING:
>> attempting to use the extensible SDK in an environment set up to run
>> bitbake - this may lead to unexpected results. Please source this script in
>> a new shell session instead.') || true" >> $e + done
>>
>> if [ "$prepare_buildsystem" != "no" ]; then
>> - printf "Preparing build system...\n"
>> + echo "Preparing build system..."
>
> Why did you change this?
We don't need complex format here, so so echo is simpler ?
// Robert
>
>> # dash which is /bin/sh on Ubuntu will not preserve the
>> # current working directory when first ran, nor will it set $1 when
>> # sourcing a script. That is why this has to look so ugly.
>> LOGFILE="$target_sdk_dir/preparing_build_system.log"
>> - sh -c ". buildtools/environment-setup* > $LOGFILE && cd
>> $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir
>> && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE
>> && python $target_sdk_dir/ext-sdk-prepare.py $LOGFILE
>> '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully
>> installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
>> - rm $target_sdk_dir/ext-sdk-prepare.py
>> + sh -c ". buildtools/environment-setup* > $LOGFILE && cd
>> $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir
>> && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE
>> && python $target_sdk_dir/ext-sdk-prepare.py $LOGFILE
>> '${SDK_INSTALL_TARGETS}'" + if [ $? -ne 0 ]; then
>> + for e in $env_setup_scripts; do
>> + echo "echo 'ERROR: this SDK was not fully installed and needs
>> reinstalling'" >> $e + done
>> + exit 1
>> + fi
>> + rm -f $target_sdk_dir/ext-sdk-prepare.py
>
> That last line is also unrelated.
>
> Cheers,
> Paul
>
^ permalink raw reply
* Re: [PATCH 5/8] oeqa/sdkext/devtool.py: don't reset when the test is failed
From: Robert Yang @ 2016-12-13 6:01 UTC (permalink / raw)
To: Paul Eggleton, openembedded-core
In-Reply-To: <4153349.TypEQkvbtP@peggleto-mobl.ger.corp.intel.com>
On 12/13/2016 12:48 PM, Paul Eggleton wrote:
> On Wed, 16 Nov 2016 22:19:34 Robert Yang wrote:
>> The contents are helpful to debug when the error happens.
>
> In this case removing this is OK because we're operating on an eSDK that the
> tests install. However, this wouldn't be appropriate for the devtool oe-
> selftest tests since they are operating on the user's build system itself -
> not that you probably have any intention of changing those, I just wanted to
> note that.
Sorry, I don't quite understand, does selftest uses oeqa/sdkext/devtool.py,
please ? Even if it uses devtool.py, I still think that we need keep the error
contents for debugging when the error happens. It's very hard to debug when
there is no error log or no workspace.
// Robert
>
> Cheers,
> Paul
>
^ permalink raw reply
* Re: [PATCH 2/8] oeqa/sdkext/devtool.py: remove workspace/sources before running test cases
From: Robert Yang @ 2016-12-13 5:56 UTC (permalink / raw)
To: Paul Eggleton
Cc: Francisco Pedraza, Limon, Anibal, Lopez, Mariano,
openembedded-core
In-Reply-To: <3281082.n0HP4LhUSl@peggleto-mobl.ger.corp.intel.com>
Hi Paul,
Thanks for reply, please see my comments inline.
On 12/13/2016 12:45 PM, Paul Eggleton wrote:
> On Wed, 16 Nov 2016 22:19:31 Robert Yang wrote:
>> Fixed:
>> MACHINE = "qemux86-64"
>> require conf/multilib.conf
>> MULTILIBS = "multilib:lib32"
>> DEFAULTTUNE_virtclass-multilib-lib32 = "x86"
>>
>> $ bitbake core-image-minimal -cpopulate_sdk_ext
>> [snip]
>> ERROR: Source tree path
>> /path/to/tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/testsdkex
>> t/tc/workspace/sources/v4l2loopback-driver already exists and is not
>> empty\n' [snip]
>>
>> This is because the test case will run twice
>> (environment-setup-core2-64-poky-linux and
>> environment-setup-x86-pokymllib32-linux), it would fail in the second
>> run, 'devtool reset' can not remove sources, so remove it before running
>> test cases.
>>
>> [YOCTO #10647]
>>
>> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
>> ---
>> meta/lib/oeqa/sdkext/devtool.py | 3 +++
>> 1 file changed, 3 insertions(+)
>>
>> diff --git a/meta/lib/oeqa/sdkext/devtool.py
>> b/meta/lib/oeqa/sdkext/devtool.py index 65f41f6..f101eb6 100644
>> --- a/meta/lib/oeqa/sdkext/devtool.py
>> +++ b/meta/lib/oeqa/sdkext/devtool.py
>> @@ -15,6 +15,9 @@ class DevtoolTest(oeSDKExtTest):
>> self.myapp_cmake_dst = os.path.join(self.tc.sdktestdir,
>> "myapp_cmake") shutil.copytree(self.myapp_cmake_src, self.myapp_cmake_dst)
>>
>> + # Clean sources dir to make "git clone" can run again
>> + shutil.rmtree(os.path.join(self.tc.sdktestdir,
>> "tc/workspace/sources"), True) +
>> def _test_devtool_build(self, directory):
>> self._run('devtool add myapp %s' % directory)
>> try:
>
> It seems to me that's what's missing here is a proper teardown process like we
> have for oe-selftest, so that tests clean up after themselves whether they
> succeed or fail. I'm unsure as to whether that is part of the plan for the new
> QA refactoring though.
There is already a 'devtool reset' which can do the cleanup, but it
can't remove sources as I said in the commit log.
// Robert
>
> In the absence of that however I guess we don't have much choice but to do
> something like this.
>
> Cheers,
> Paul
>
^ permalink raw reply
* Re: [PATCH] glibc: add -fno-builtin-strlen when not using -O2
From: Huang, Jie (Jackie) @ 2016-12-13 5:14 UTC (permalink / raw)
To: Andre McCurdy; +Cc: Patches and discussions about the oe-core layer
In-Reply-To: <CAJ86T=Ux+uT-PgJ+H+-i60iABaQ3iuUB8gz6iLF-+vtMycZdWw@mail.gmail.com>
> -----Original Message-----
> From: Andre McCurdy [mailto:armccurdy@gmail.com]
> Sent: Monday, December 12, 2016 5:55 PM
> To: Huang, Jie (Jackie)
> Cc: Khem Raj; Patches and discussions about the oe-core layer
> Subject: Re: [OE-core] [PATCH] glibc: add -fno-builtin-strlen when not using -O2
>
> On Sun, Dec 11, 2016 at 11:41 PM, Huang, Jie (Jackie)
> <Jackie.Huang@windriver.com> wrote:
> >> -----Original Message-----
> >> From: Khem Raj [mailto:raj.khem@gmail.com]
> >> Sent: Monday, December 12, 2016 2:37 PM
> >> To: Huang, Jie (Jackie)
> >> Cc: Patches and discussions about the oe-core layer
> >> Subject: Re: [OE-core] [PATCH] glibc: add -fno-builtin-strlen when not using -O2
> >>
> >> On Sun, Dec 11, 2016 at 9:42 PM, <jackie.huang@windriver.com> wrote:
> >> > From: Jackie Huang <jackie.huang@windriver.com>
> >> >
> >> > The strlen will be inlined when compile with -O, -O1 or -Os,
> >> > so there is no symbol for strlen in ld-linux-x86-64.so.2,
> >> > causing a fatal error in valgrind:
> >> >
> >> > valgrind: Fatal error at startup: a function redirection
> >> > valgrind: which is mandatory for this platform-tool combination
> >> > valgrind: cannot be set up. Details of the redirection are:
> >> > valgrind:
> >> > valgrind: A must-be-redirected function
> >> > valgrind: whose name matches the pattern: strlen
> >> > valgrind: in an object with soname matching: ld-linux-x86-64.so.2
> >> >
> >> > so add -fno-builtin-strlen when compile with -O, -O1 or -Os.
> >>
> >> This is a bug in valgrind, I read the same on forums. KDE has proposed a fix
> >> https://bugsfiles.kde.org/attachment.cgi?id=82043
> >> can you check if this fixes the issue
> >
> > Thanks, I will check that and change to patch for valgrind if it works.
>
> For reference, here's the patch I've been using. It's a slightly more
> generic fix than the one in the KDE bug report.
Thanks, It's a better patch and I will take it and send as v2 of this issue if you're
not going to send it yourself, is it fine for you and could you provide extra info
for the patch header like, upstream-status, written by or Signed-off-by?
Thanks,
Jackie
>
> diff --git a/coregrind/m_redir.c b/coregrind/m_redir.c
> index 7e4df8d..640a346 100644
> --- a/coregrind/m_redir.c
> +++ b/coregrind/m_redir.c
> @@ -1220,7 +1220,18 @@ static void add_hardwired_spec (const HChar*
> sopatt, const HChar* fnpatt,
> spec->from_fnpatt = CONST_CAST(HChar *,fnpatt);
> spec->to_addr = to_addr;
> spec->isWrap = False;
> - spec->mandatory = mandatory;
> +
> + /* Hack: Depending on how glibc was compiled (e.g. optimised for size or
> + built with _FORTIFY_SOURCE enabled) the strlen symbol might not be found.
> + Therefore although we should still try to intercept it, don't make it
> + mandatory to do so. We over-ride "mandatory" here to avoid the need to
> + patch the many different architecture specific callers to
> + add_hardwired_spec(). */
> + if (0==VG_(strcmp)("strlen", fnpatt))
> + spec->mandatory = NULL;
> + else
> + spec->mandatory = mandatory;
> +
> /* VARIABLE PARTS */
> spec->mark = False; /* not significant */
> spec->done = False; /* not significant */
^ permalink raw reply
* Re: [PATCH 4/8] oe-publish-sdk: add pyshtables.py to .gitignore
From: Paul Eggleton @ 2016-12-13 4:59 UTC (permalink / raw)
To: Robert Yang; +Cc: openembedded-core
In-Reply-To: <f7797b9e72ed6e6f7e05be8868b73c68a4ee1a79.1479363545.git.liezhi.yang@windriver.com>
On Wed, 16 Nov 2016 22:19:33 Robert Yang wrote:
> Fixed:
> MACHINE = "qemux86-64"
> require conf/multilib.conf
> MULTILIBS = "multilib:lib32"
> DEFAULTTUNE_virtclass-multilib-lib32 = "x86"
>
> $ bitbake core-image-minimal -cpopulate_sdk_ext
> [snip]
> ERROR: Failed to update metadata as there have been changes made to it.
> Aborting.\nERROR: Changed files:\nb' M
> poky/bitbake/lib/bb/pysh/pyshtables.py\\n'\n" [snip]
>
> This is because the test case will run twice
> (environment-setup-core2-64-poky-linux and
> environment-setup-x86-pokymllib32-linux), it would fail in the second
> run since pyshtables.py is regenerated in the first run. This file is
> generated automatically, publish it doesn't make any sense, so add it to
> .gitignore.
>
> [YOCTO #10647]
The actual fix looks OK but I don't think 10647 is really the right bug
number. Coincidentally today someone else reported this issue, so this would
probably be a better one:
https://bugzilla.yoctoproject.org/show_bug.cgi?id=10796
> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
> ---
> scripts/oe-publish-sdk | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/scripts/oe-publish-sdk b/scripts/oe-publish-sdk
> index d95c623..e2b1b95 100755
> --- a/scripts/oe-publish-sdk
> +++ b/scripts/oe-publish-sdk
> @@ -116,7 +116,7 @@ def publish(args):
> cmd_common = "if [ ! -e .git ]; then"
> cmd_common += " git init .;"
> cmd_common += " mv .git/hooks/post-update.sample
> .git/hooks/post-update;" - cmd_common += " echo '*.pyc\n*.pyo' >
> .gitignore;"
> + cmd_common += " echo '*.pyc\n*.pyo\npyshtables.py' > .gitignore;"
> cmd_common += "fi;"
> cmd_common += "git add -A .;"
> cmd_common += "git config user.email 'oe@oe.oe' && git config user.name
> 'OE' && git commit -q -m 'init repo' || true;"
Cheers,
Paul
--
Paul Eggleton
Intel Open Source Technology Centre
^ permalink raw reply
* Re: [PATCH 1/8] populate_sdk_ext.bbclass: install multilib targets as populate_sdk does
From: Paul Eggleton @ 2016-12-13 4:55 UTC (permalink / raw)
To: Robert Yang; +Cc: openembedded-core
In-Reply-To: <efeb9ec88465377d3accb9c96324a3157b1c2738.1479363545.git.liezhi.yang@windriver.com>
Hi Robert,
There are a bunch of changes in here that don't relate to the multilib fix -
details below.
On Wed, 16 Nov 2016 22:19:30 Robert Yang wrote:
> Fixed:
> MACHINE = "qemux86-64"
> require conf/multilib.conf
> MULTILIBS = "multilib:lib32"
> DEFAULTTUNE_virtclass-multilib-lib32 = "x86"
>
> $ bitbake core-image-minimal -cpopulate_sdk_ext
> [snip]
> Testing
> /buildarea/lyang1/test_po/tmp/work/qemux86_64-poky-linux/core-image-minimal
> /1.0-r0/testsdkext//tc/environment-setup-x86-pokymllib32-linux test_cvs
> (oeqa.sdk.buildcvs.BuildCvsTest) ... FAIL
> [snip]
>
> It was failed because no lib32 toolchains.
>
> The fixes include:
> * Set SDK_TARGETS correctly
> * Return multilib depends in get_ext_sdk_depends()
> * Write information to all environment-setup-* scripts.
>
> [YOCTO #10647]
>
> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
> ---
> meta/classes/populate_sdk_ext.bbclass | 61
> ++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 23
> deletions(-)
>
> diff --git a/meta/classes/populate_sdk_ext.bbclass
> b/meta/classes/populate_sdk_ext.bbclass index 26b5ca6..ce9c40a 100644
> --- a/meta/classes/populate_sdk_ext.bbclass
> +++ b/meta/classes/populate_sdk_ext.bbclass
> @@ -39,7 +39,7 @@ SDK_LOCAL_CONF_BLACKLIST ?= "CONF_VERSION \
> SDK_INHERIT_BLACKLIST ?= "buildhistory icecc"
> SDK_UPDATE_URL ?= ""
>
> -SDK_TARGETS ?= "${PN}"
> +SDK_TARGETS ?= "${@multilib_pkg_extend(d, d.getVar('BPN', True))}"
>
> def get_sdk_install_targets(d, images_only=False):
> sdk_install_targets = ''
> @@ -562,38 +562,52 @@ SDK_PRE_INSTALL_COMMAND_task-populate-sdk-ext =
> "${sdk_ext_preinst}" sdk_ext_postinst() {
> printf "\nExtracting buildtools...\n"
> cd $target_sdk_dir
> - env_setup_script="$target_sdk_dir/environment-
setup-${REAL_MULTIMACH_TARGE
> T_SYS}" - printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} >
> buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat
> buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and
> needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
> + env_setup_scripts="`ls $target_sdk_dir/environment-setup-*`"
> + ./${SDK_BUILDTOOLS_INSTALLER} -d buildtools -y > buildtools.log
> + if [ $? -ne 0 ]; then
> + echo 'ERROR: buildtools installation failed:'
> + cat buildtools.log
> + for e in $env_setup_scripts; do
> + echo "echo 'ERROR: this SDK was not fully installed and needs
> reinstalling'" >> $e + done
> + exit 1
> + fi
>
This change isn't entirely related to the multilib changes.
> # Delete the buildtools tar file since it won't be used again
> rm -f ./${SDK_BUILDTOOLS_INSTALLER}
> # We don't need the log either since it succeeded
> rm -f buildtools.log
>
> - # Make sure when the user sets up the environment, they also get
> - # the buildtools-tarball tools in their path.
> - echo ". $target_sdk_dir/buildtools/environment-setup*" >>
> $env_setup_script -
> - # Allow bitbake environment setup to be ran as part of this sdk.
> - echo "export OE_SKIP_SDK_CHECK=1" >> $env_setup_script
> + for e in $env_setup_scripts; do
> + # Make sure when the user sets up the environment, they also get
> + # the buildtools-tarball tools in their path.
> + echo ". $target_sdk_dir/buildtools/environment-setup*" >> $e
>
> - # A bit of another hack, but we need this in the path only for devtool
> - # so put it at the end of $PATH.
> - echo "export
> PATH=$target_sdk_dir/sysroots/${SDK_SYS}${bindir_nativesdk}:\$PATH" >>
> $env_setup_script + # Allow bitbake environment setup to be ran as part
of
> this sdk. + echo "export OE_SKIP_SDK_CHECK=1" >> $e
>
> - echo "printf 'SDK environment now set up; additionally you may now run
> devtool to perform development tasks.\nRun devtool --help for further
> details.\n'" >> $env_setup_script -
> - # Warn if trying to use external bitbake and the ext SDK together
> - echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING: attempting to use
> the extensible SDK in an environment set up to run bitbake - this may lead
> to unexpected results. Please source this script in a new shell session
> instead.') || true" >> $env_setup_script + # A bit of another hack, but
we
> need this in the path only for devtool + # so put it at the end of
$PATH.
> + echo "export
> PATH=$target_sdk_dir/sysroots/${SDK_SYS}${bindir_nativesdk}:\$PATH" >> $e
> + echo "printf 'SDK environment now set up; additionally you may now
run
> devtool to perform development tasks.\nRun devtool --help for further
> details.\n'" >> $e + # Warn if trying to use external bitbake and the
ext
> SDK together + echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING:
> attempting to use the extensible SDK in an environment set up to run
> bitbake - this may lead to unexpected results. Please source this script in
> a new shell session instead.') || true" >> $e + done
>
> if [ "$prepare_buildsystem" != "no" ]; then
> - printf "Preparing build system...\n"
> + echo "Preparing build system..."
Why did you change this?
> # dash which is /bin/sh on Ubuntu will not preserve the
> # current working directory when first ran, nor will it set $1 when
> # sourcing a script. That is why this has to look so ugly.
> LOGFILE="$target_sdk_dir/preparing_build_system.log"
> - sh -c ". buildtools/environment-setup* > $LOGFILE && cd
> $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir
> && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE
> && python $target_sdk_dir/ext-sdk-prepare.py $LOGFILE
> '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully
> installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
> - rm $target_sdk_dir/ext-sdk-prepare.py
> + sh -c ". buildtools/environment-setup* > $LOGFILE && cd
> $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir
> && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE
> && python $target_sdk_dir/ext-sdk-prepare.py $LOGFILE
> '${SDK_INSTALL_TARGETS}'" + if [ $? -ne 0 ]; then
> + for e in $env_setup_scripts; do
> + echo "echo 'ERROR: this SDK was not fully installed and needs
> reinstalling'" >> $e + done
> + exit 1
> + fi
> + rm -f $target_sdk_dir/ext-sdk-prepare.py
That last line is also unrelated.
Cheers,
Paul
--
Paul Eggleton
Intel Open Source Technology Centre
^ permalink raw reply
* Re: [PATCH 5/8] oeqa/sdkext/devtool.py: don't reset when the test is failed
From: Paul Eggleton @ 2016-12-13 4:48 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <5cea51d1030f17c4c8fecfb4da3b04bd63a2b6c2.1479363545.git.liezhi.yang@windriver.com>
On Wed, 16 Nov 2016 22:19:34 Robert Yang wrote:
> The contents are helpful to debug when the error happens.
In this case removing this is OK because we're operating on an eSDK that the
tests install. However, this wouldn't be appropriate for the devtool oe-
selftest tests since they are operating on the user's build system itself -
not that you probably have any intention of changing those, I just wanted to
note that.
Cheers,
Paul
--
Paul Eggleton
Intel Open Source Technology Centre
^ permalink raw reply
* Re: [PATCH 2/8] oeqa/sdkext/devtool.py: remove workspace/sources before running test cases
From: Paul Eggleton @ 2016-12-13 4:45 UTC (permalink / raw)
To: Robert Yang
Cc: Francisco Pedraza, Limon, Anibal, Lopez, Mariano,
openembedded-core
In-Reply-To: <61246658fe264929d272f4da2f681e0fb95f9809.1479363545.git.liezhi.yang@windriver.com>
On Wed, 16 Nov 2016 22:19:31 Robert Yang wrote:
> Fixed:
> MACHINE = "qemux86-64"
> require conf/multilib.conf
> MULTILIBS = "multilib:lib32"
> DEFAULTTUNE_virtclass-multilib-lib32 = "x86"
>
> $ bitbake core-image-minimal -cpopulate_sdk_ext
> [snip]
> ERROR: Source tree path
> /path/to/tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/testsdkex
> t/tc/workspace/sources/v4l2loopback-driver already exists and is not
> empty\n' [snip]
>
> This is because the test case will run twice
> (environment-setup-core2-64-poky-linux and
> environment-setup-x86-pokymllib32-linux), it would fail in the second
> run, 'devtool reset' can not remove sources, so remove it before running
> test cases.
>
> [YOCTO #10647]
>
> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
> ---
> meta/lib/oeqa/sdkext/devtool.py | 3 +++
> 1 file changed, 3 insertions(+)
>
> diff --git a/meta/lib/oeqa/sdkext/devtool.py
> b/meta/lib/oeqa/sdkext/devtool.py index 65f41f6..f101eb6 100644
> --- a/meta/lib/oeqa/sdkext/devtool.py
> +++ b/meta/lib/oeqa/sdkext/devtool.py
> @@ -15,6 +15,9 @@ class DevtoolTest(oeSDKExtTest):
> self.myapp_cmake_dst = os.path.join(self.tc.sdktestdir,
> "myapp_cmake") shutil.copytree(self.myapp_cmake_src, self.myapp_cmake_dst)
>
> + # Clean sources dir to make "git clone" can run again
> + shutil.rmtree(os.path.join(self.tc.sdktestdir,
> "tc/workspace/sources"), True) +
> def _test_devtool_build(self, directory):
> self._run('devtool add myapp %s' % directory)
> try:
It seems to me that's what's missing here is a proper teardown process like we
have for oe-selftest, so that tests clean up after themselves whether they
succeed or fail. I'm unsure as to whether that is part of the plan for the new
QA refactoring though.
In the absence of that however I guess we don't have much choice but to do
something like this.
Cheers,
Paul
--
Paul Eggleton
Intel Open Source Technology Centre
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox