* [PATCH 1/7] devtool: update-recipe: find and use existing files directories
2016-07-13 21:04 [PATCH 0/7] New devtool finish subcommand Paul Eggleton
@ 2016-07-13 21:04 ` Paul Eggleton
2016-07-13 21:04 ` [PATCH 2/7] devtool: update-recipe: don't copy local files that haven't changed Paul Eggleton
` (5 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Paul Eggleton @ 2016-07-13 21:04 UTC (permalink / raw)
To: openembedded-core
devtool update-recipe was defaulting to the ${BPN} named directory when
adding patches next to a recipe, but that meant if you already had files
in a ${BP} named directory (i.e. name and version) or "files" then you'd
end up with two directories next to the recipe, which is usually not
what you want. To avoid this, look through FILESPATH and take the first
one that's the same level or one level down from the recipe and already
exists, if any.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
scripts/lib/devtool/standard.py | 18 ++++++++++++++----
1 file changed, 14 insertions(+), 4 deletions(-)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index f2ba699..1ec31bc 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -1039,6 +1039,18 @@ def _export_local_files(srctree, rd, destdir):
return (updated, added, removed)
+def _determine_files_dir(rd):
+ """Determine the appropriate files directory for a recipe"""
+ recipedir = rd.getVar('FILE_DIRNAME', True)
+ for entry in rd.getVar('FILESPATH', True).split(':'):
+ relpth = os.path.relpath(entry, recipedir)
+ if not os.sep in relpth:
+ # One (or zero) levels below only, so we don't put anything in machine-specific directories
+ if os.path.isdir(entry):
+ return entry
+ return os.path.join(recipedir, rd.getVar('BPN', True))
+
+
def _update_recipe_srcrev(args, srctree, rd, config_data):
"""Implement the 'srcrev' mode of update-recipe"""
import bb
@@ -1092,8 +1104,7 @@ def _update_recipe_srcrev(args, srctree, rd, config_data):
rd, args.append, files, wildcardver=args.wildcard_version,
extralines=patchfields, removevalues=removevalues)
else:
- files_dir = os.path.join(os.path.dirname(recipefile),
- rd.getVar('BPN', True))
+ files_dir = _determine_files_dir(rd)
for basepath, path in upd_f.items():
logger.info('Updating file %s' % basepath)
_move_file(os.path.join(local_files_dir, basepath), path)
@@ -1193,8 +1204,7 @@ def _update_recipe_patch(args, config, workspace, srctree, rd, config_data):
_move_file(patchfn, path)
updatefiles = True
# Add any new files
- files_dir = os.path.join(os.path.dirname(recipefile),
- rd.getVar('BPN', True))
+ files_dir = _determine_files_dir(rd)
for basepath, path in new_f.items():
logger.info('Adding new file %s' % basepath)
_move_file(os.path.join(local_files_dir, basepath),
--
2.5.5
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 2/7] devtool: update-recipe: don't copy local files that haven't changed
2016-07-13 21:04 [PATCH 0/7] New devtool finish subcommand Paul Eggleton
2016-07-13 21:04 ` [PATCH 1/7] devtool: update-recipe: find and use existing files directories Paul Eggleton
@ 2016-07-13 21:04 ` Paul Eggleton
2016-07-13 21:04 ` [PATCH 3/7] lib/oe/recipeutils: fix a few issues in find_layerdir() Paul Eggleton
` (4 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Paul Eggleton @ 2016-07-13 21:04 UTC (permalink / raw)
To: openembedded-core
If there are files in the oe-local-files directory which are identical
to the original version, then we shouldn't be copying them to the
destination layer. This is particularly important when using the -a
option to create a bbappend.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
scripts/lib/devtool/standard.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 1ec31bc..e198120 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -28,6 +28,7 @@ import argparse_oe
import scriptutils
import errno
import glob
+import filecmp
from collections import OrderedDict
from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, DevtoolError
from devtool import parse_recipe
@@ -1031,7 +1032,10 @@ def _export_local_files(srctree, rd, destdir):
if new_set is not None:
for fname in new_set:
if fname in existing_files:
- updated[fname] = existing_files.pop(fname)
+ origpath = existing_files.pop(fname)
+ workpath = os.path.join(local_files_dir, fname)
+ if not filecmp.cmp(origpath, workpath):
+ updated[fname] = origpath
elif fname != '.gitignore':
added[fname] = None
--
2.5.5
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 3/7] lib/oe/recipeutils: fix a few issues in find_layerdir()
2016-07-13 21:04 [PATCH 0/7] New devtool finish subcommand Paul Eggleton
2016-07-13 21:04 ` [PATCH 1/7] devtool: update-recipe: find and use existing files directories Paul Eggleton
2016-07-13 21:04 ` [PATCH 2/7] devtool: update-recipe: don't copy local files that haven't changed Paul Eggleton
@ 2016-07-13 21:04 ` Paul Eggleton
2016-07-13 21:04 ` [PATCH 4/7] devtool: reset: refactor to allow calling separately Paul Eggleton
` (3 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Paul Eggleton @ 2016-07-13 21:04 UTC (permalink / raw)
To: openembedded-core
* Allow the function to be called with the base layer path (in which
case it will just return the same path)
* Ensure that the function doesn't recurse indefinitely if it's called
on a file that's not inside a layer
* Correct the doc comment for accuracy
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
meta/lib/oe/recipeutils.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index e3c4b8a..cb4ed53 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -728,14 +728,16 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
def find_layerdir(fn):
- """ Figure out relative path to base of layer for a file (e.g. a recipe)"""
- pth = os.path.dirname(fn)
+ """ Figure out the path to the base of the layer containing a file (e.g. a recipe)"""
+ pth = fn
layerdir = ''
while pth:
if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')):
layerdir = pth
break
pth = os.path.dirname(pth)
+ if pth == '/':
+ return None
return layerdir
--
2.5.5
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 4/7] devtool: reset: refactor to allow calling separately
2016-07-13 21:04 [PATCH 0/7] New devtool finish subcommand Paul Eggleton
` (2 preceding siblings ...)
2016-07-13 21:04 ` [PATCH 3/7] lib/oe/recipeutils: fix a few issues in find_layerdir() Paul Eggleton
@ 2016-07-13 21:04 ` Paul Eggleton
2016-07-13 21:04 ` [PATCH 5/7] devtool: update-recipe: " Paul Eggleton
` (2 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Paul Eggleton @ 2016-07-13 21:04 UTC (permalink / raw)
To: openembedded-core
This will be called by "devtool finish" to allow it to reset the recipe
at the end.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
scripts/lib/devtool/standard.py | 40 +++++++++++++++++++++++-----------------
1 file changed, 23 insertions(+), 17 deletions(-)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index e198120..769c3ce 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -1314,24 +1314,10 @@ def status(args, config, basepath, workspace):
return 0
-def reset(args, config, basepath, workspace):
- """Entry point for the devtool 'reset' subcommand"""
- import bb
- if args.recipename:
- if args.all:
- raise DevtoolError("Recipe cannot be specified if -a/--all is used")
- else:
- for recipe in args.recipename:
- check_workspace_recipe(workspace, recipe, checksrc=False)
- elif not args.all:
- raise DevtoolError("Recipe must be specified, or specify -a/--all to "
- "reset all recipes")
- if args.all:
- recipes = list(workspace.keys())
- else:
- recipes = args.recipename
+def _reset(recipes, no_clean, config, basepath, workspace):
+ """Reset one or more recipes"""
- if recipes and not args.no_clean:
+ if recipes and not no_clean:
if len(recipes) == 1:
logger.info('Cleaning sysroot for recipe %s...' % recipes[0])
else:
@@ -1383,6 +1369,26 @@ def reset(args, config, basepath, workspace):
# This is unlikely, but if it's empty we can just remove it
os.rmdir(srctree)
+
+def reset(args, config, basepath, workspace):
+ """Entry point for the devtool 'reset' subcommand"""
+ import bb
+ if args.recipename:
+ if args.all:
+ raise DevtoolError("Recipe cannot be specified if -a/--all is used")
+ else:
+ for recipe in args.recipename:
+ check_workspace_recipe(workspace, recipe, checksrc=False)
+ elif not args.all:
+ raise DevtoolError("Recipe must be specified, or specify -a/--all to "
+ "reset all recipes")
+ if args.all:
+ recipes = list(workspace.keys())
+ else:
+ recipes = args.recipename
+
+ _reset(recipes, args.no_clean, config, basepath, workspace)
+
return 0
--
2.5.5
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 5/7] devtool: update-recipe: refactor to allow calling separately
2016-07-13 21:04 [PATCH 0/7] New devtool finish subcommand Paul Eggleton
` (3 preceding siblings ...)
2016-07-13 21:04 ` [PATCH 4/7] devtool: reset: refactor to allow calling separately Paul Eggleton
@ 2016-07-13 21:04 ` Paul Eggleton
2016-07-13 21:04 ` [PATCH 6/7] devtool: upgrade: record original recipe files Paul Eggleton
2016-07-13 21:04 ` [PATCH 7/7] devtool: add finish subcommand Paul Eggleton
6 siblings, 0 replies; 8+ messages in thread
From: Paul Eggleton @ 2016-07-13 21:04 UTC (permalink / raw)
To: openembedded-core
This will be called by "devtool finish" to allow it to update the recipe
or create the bbappend depending on the destination.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
scripts/lib/devtool/standard.py | 56 +++++++++++++++++++++--------------------
1 file changed, 29 insertions(+), 27 deletions(-)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 769c3ce..5a5995f 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -876,10 +876,10 @@ def _remove_file_entries(srcuri, filelist):
break
return entries, remaining
-def _remove_source_files(args, files, destpath):
+def _remove_source_files(append, files, destpath):
"""Unlink existing patch files"""
for path in files:
- if args.append:
+ if append:
if not destpath:
raise Exception('destpath should be set here')
path = os.path.join(destpath, os.path.basename(path))
@@ -1055,7 +1055,7 @@ def _determine_files_dir(rd):
return os.path.join(recipedir, rd.getVar('BPN', True))
-def _update_recipe_srcrev(args, srctree, rd, config_data):
+def _update_recipe_srcrev(srctree, rd, appendlayerdir, wildcard_version, no_remove):
"""Implement the 'srcrev' mode of update-recipe"""
import bb
import oe.recipeutils
@@ -1084,7 +1084,7 @@ def _update_recipe_srcrev(args, srctree, rd, config_data):
try:
local_files_dir = tempfile.mkdtemp(dir=tempdir)
upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir)
- if not args.no_remove:
+ if not no_remove:
# Find list of existing patches in recipe file
patches_dir = tempfile.mkdtemp(dir=tempdir)
old_srcrev = (rd.getVar('SRCREV', False) or '')
@@ -1097,7 +1097,7 @@ def _update_recipe_srcrev(args, srctree, rd, config_data):
removedentries = _remove_file_entries(srcuri, remove_files)[0]
update_srcuri = True
- if args.append:
+ if appendlayerdir:
files = dict((os.path.join(local_files_dir, key), val) for
key, val in list(upd_f.items()) + list(new_f.items()))
removevalues = {}
@@ -1105,7 +1105,7 @@ def _update_recipe_srcrev(args, srctree, rd, config_data):
removevalues = {'SRC_URI': removedentries}
patchfields['SRC_URI'] = '\\\n '.join(srcuri)
_, destpath = oe.recipeutils.bbappend_recipe(
- rd, args.append, files, wildcardver=args.wildcard_version,
+ rd, appendlayerdir, files, wildcardver=wildcard_version,
extralines=patchfields, removevalues=removevalues)
else:
files_dir = _determine_files_dir(rd)
@@ -1129,21 +1129,21 @@ def _update_recipe_srcrev(args, srctree, rd, config_data):
'point to a git repository where you have pushed your '
'changes')
- _remove_source_files(args, remove_files, destpath)
+ _remove_source_files(appendlayerdir, remove_files, destpath)
return True
-def _update_recipe_patch(args, config, workspace, srctree, rd, config_data):
+def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, initial_rev):
"""Implement the 'patch' mode of update-recipe"""
import bb
import oe.recipeutils
recipefile = rd.getVar('FILE', True)
- append = workspace[args.recipename]['bbappend']
+ append = workspace[recipename]['bbappend']
if not os.path.exists(append):
raise DevtoolError('unable to find workspace bbappend for recipe %s' %
- args.recipename)
+ recipename)
- initial_rev, update_rev, changed_revs = _get_patchset_revs(srctree, append, args.initial_rev)
+ initial_rev, update_rev, changed_revs = _get_patchset_revs(srctree, append, initial_rev)
if not initial_rev:
raise DevtoolError('Unable to find initial revision - please specify '
'it with --initial-rev')
@@ -1154,7 +1154,7 @@ def _update_recipe_patch(args, config, workspace, srctree, rd, config_data):
upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir)
remove_files = []
- if not args.no_remove:
+ if not no_remove:
# Get all patches from source tree and check if any should be removed
all_patches_dir = tempfile.mkdtemp(dir=tempdir)
upd_p, new_p, del_p = _export_patches(srctree, rd, initial_rev,
@@ -1170,7 +1170,7 @@ def _update_recipe_patch(args, config, workspace, srctree, rd, config_data):
updaterecipe = False
destpath = None
srcuri = (rd.getVar('SRC_URI', False) or '').split()
- if args.append:
+ if appendlayerdir:
files = dict((os.path.join(local_files_dir, key), val) for
key, val in list(upd_f.items()) + list(new_f.items()))
files.update(dict((os.path.join(patches_dir, key), val) for
@@ -1185,7 +1185,7 @@ def _update_recipe_patch(args, config, workspace, srctree, rd, config_data):
item in remaining]
removevalues = {'SRC_URI': removedentries + remaining}
_, destpath = oe.recipeutils.bbappend_recipe(
- rd, args.append, files,
+ rd, appendlayerdir, files,
removevalues=removevalues)
else:
logger.info('No patches or local source files needed updating')
@@ -1235,7 +1235,7 @@ def _update_recipe_patch(args, config, workspace, srctree, rd, config_data):
finally:
shutil.rmtree(tempdir)
- _remove_source_files(args, remove_files, destpath)
+ _remove_source_files(appendlayerdir, remove_files, destpath)
return True
def _guess_recipe_update_mode(srctree, rdata):
@@ -1260,6 +1260,19 @@ def _guess_recipe_update_mode(srctree, rdata):
return 'patch'
+def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev):
+ srctree = workspace[recipename]['srctree']
+ if mode == 'auto':
+ mode = _guess_recipe_update_mode(srctree, rd)
+
+ if mode == 'srcrev':
+ updated = _update_recipe_srcrev(srctree, rd, appendlayerdir, wildcard_version, no_remove)
+ elif mode == 'patch':
+ updated = _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, initial_rev)
+ else:
+ raise DevtoolError('update_recipe: invalid mode %s' % mode)
+ return updated
+
def update_recipe(args, config, basepath, workspace):
"""Entry point for the devtool 'update-recipe' subcommand"""
check_workspace_recipe(workspace, args.recipename)
@@ -1278,18 +1291,7 @@ def update_recipe(args, config, basepath, workspace):
if not rd:
return 1
- srctree = workspace[args.recipename]['srctree']
- if args.mode == 'auto':
- mode = _guess_recipe_update_mode(srctree, rd)
- else:
- mode = args.mode
-
- if mode == 'srcrev':
- updated = _update_recipe_srcrev(args, srctree, rd, tinfoil.config_data)
- elif mode == 'patch':
- updated = _update_recipe_patch(args, config, workspace, srctree, rd, tinfoil.config_data)
- else:
- raise DevtoolError('update_recipe: invalid mode %s' % mode)
+ updated = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev)
if updated:
rf = rd.getVar('FILE', True)
--
2.5.5
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 6/7] devtool: upgrade: record original recipe files
2016-07-13 21:04 [PATCH 0/7] New devtool finish subcommand Paul Eggleton
` (4 preceding siblings ...)
2016-07-13 21:04 ` [PATCH 5/7] devtool: update-recipe: " Paul Eggleton
@ 2016-07-13 21:04 ` Paul Eggleton
2016-07-13 21:04 ` [PATCH 7/7] devtool: add finish subcommand Paul Eggleton
6 siblings, 0 replies; 8+ messages in thread
From: Paul Eggleton @ 2016-07-13 21:04 UTC (permalink / raw)
To: openembedded-core
This provides us with the information we need to remove the original
version recipe and associated files when running "devtool finish" after
"devtool upgrade".
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
meta/lib/oe/recipeutils.py | 4 +++-
scripts/lib/devtool/upgrade.py | 16 ++++++++++------
2 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index cb4ed53..b8d481a 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -365,6 +365,7 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True):
# Copy local files to target directory and gather any remote files
bb_dir = os.path.dirname(d.getVar('FILE', True)) + os.sep
remotes = []
+ copied = []
includes = [path for path in d.getVar('BBINCLUDED', True).split() if
path.startswith(bb_dir) and os.path.exists(path)]
for path in fetch.localpaths() + includes:
@@ -376,13 +377,14 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True):
if not os.path.exists(subdir):
os.makedirs(subdir)
shutil.copy2(path, os.path.join(tgt_dir, relpath))
+ copied.append(relpath)
else:
remotes.append(path)
# Simply copy whole meta dir, if requested
if whole_dir:
shutil.copytree(bb_dir, tgt_dir)
- return remotes
+ return copied, remotes
def get_recipe_local_files(d, patches=False):
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 66e5f59..8ea72ef 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -105,7 +105,7 @@ def _rename_recipe_files(oldrecipe, bpn, oldpv, newpv, path):
_rename_recipe_dirs(oldpv, newpv, path)
return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path)
-def _write_append(rc, srctree, same_dir, no_same_dir, rev, workspace, d):
+def _write_append(rc, srctree, same_dir, no_same_dir, rev, copied, workspace, d):
"""Writes an append file"""
if not os.path.exists(rc):
raise DevtoolError("bbappend not created because %s does not exist" % rc)
@@ -128,8 +128,12 @@ def _write_append(rc, srctree, same_dir, no_same_dir, rev, workspace, d):
b_is_s = use_external_build(same_dir, no_same_dir, d)
if b_is_s:
f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree))
+ f.write('\n')
if rev:
- f.write('\n# initial_rev: %s\n' % rev)
+ f.write('# initial_rev: %s\n' % rev)
+ if copied:
+ f.write('# original_path: %s\n' % os.path.dirname(d.getVar('FILE', True)))
+ f.write('# original_files: %s\n' % ' '.join(copied))
return af
def _cleanup_on_error(rf, srctree):
@@ -267,7 +271,7 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, workspace, tinfoil
bpn = rd.getVar('BPN', True)
path = os.path.join(workspace, 'recipes', bpn)
bb.utils.mkdirhier(path)
- oe.recipeutils.copy_recipe_files(rd, path)
+ copied, _ = oe.recipeutils.copy_recipe_files(rd, path)
oldpv = rd.getVar('PV', True)
if not newpv:
@@ -317,7 +321,7 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, workspace, tinfoil
rd = oe.recipeutils.parse_recipe(fullpath, None, tinfoil.config_data)
oe.recipeutils.patch_recipe(rd, fullpath, newvalues)
- return fullpath
+ return fullpath, copied
def upgrade(args, config, basepath, workspace):
"""Entry point for the devtool 'upgrade' subcommand"""
@@ -360,7 +364,7 @@ def upgrade(args, config, basepath, workspace):
rev2, md5, sha256 = _extract_new_source(args.version, srctree, args.no_patch,
args.srcrev, args.branch, args.keep_temp,
tinfoil, rd)
- rf = _create_new_recipe(args.version, md5, sha256, args.srcrev, args.srcbranch, config.workspace_path, tinfoil, rd)
+ rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, args.srcbranch, config.workspace_path, tinfoil, rd)
except bb.process.CmdError as e:
_upgrade_error(e, rf, srctree)
except DevtoolError as e:
@@ -368,7 +372,7 @@ def upgrade(args, config, basepath, workspace):
standard._add_md5(config, pn, os.path.dirname(rf))
af = _write_append(rf, srctree, args.same_dir, args.no_same_dir, rev2,
- config.workspace_path, rd)
+ copied, config.workspace_path, rd)
standard._add_md5(config, pn, af)
logger.info('Upgraded source extracted to %s' % srctree)
logger.info('New recipe is %s' % rf)
--
2.5.5
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 7/7] devtool: add finish subcommand
2016-07-13 21:04 [PATCH 0/7] New devtool finish subcommand Paul Eggleton
` (5 preceding siblings ...)
2016-07-13 21:04 ` [PATCH 6/7] devtool: upgrade: record original recipe files Paul Eggleton
@ 2016-07-13 21:04 ` Paul Eggleton
6 siblings, 0 replies; 8+ messages in thread
From: Paul Eggleton @ 2016-07-13 21:04 UTC (permalink / raw)
To: openembedded-core
Add a subcommand which will "finish" the work on a recipe. This is
effectively the same as update-recipe followed by reset, except that the
destination layer is required and it will do the right thing depending
on the situation - if the recipe file itself is in the workspace (e.g.
as a result of devtool add), the recipe file and any associated files
will be moved to the destination layer; or if the destination layer is
the one containing the original recipe, the recipe will be overwritten;
otherwise a bbappend will be created to apply the changes. In all cases
the layer path can be loosely specified - it could be a layer name, or
a partial path into a recipe. In the case of upgrades, devtool finish
will also take care of deleting the old recipe.
This avoids the user having to figure out the correct actions when
they're done - they just do "devtool finish recipename layername" and
it saves their work and then removes the recipe from the workspace.
Addresses [YOCTO #8594].
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
meta/lib/oe/recipeutils.py | 57 +++++++++++++-
meta/lib/oeqa/selftest/devtool.py | 157 +++++++++++++++++++++++++++++++++++++-
scripts/lib/devtool/standard.py | 111 ++++++++++++++++++++++++++-
3 files changed, 322 insertions(+), 3 deletions(-)
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index b8d481a..0e7abf8 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -2,7 +2,7 @@
#
# Some code borrowed from the OE layer index
#
-# Copyright (C) 2013-2015 Intel Corporation
+# Copyright (C) 2013-2016 Intel Corporation
#
import sys
@@ -15,6 +15,7 @@ from . import utils
import shutil
import re
import fnmatch
+import glob
from collections import OrderedDict, defaultdict
@@ -450,6 +451,60 @@ def validate_pn(pn):
return ''
+def get_bbfile_path(d, destdir, extrapathhint=None):
+ """
+ Determine the correct path for a recipe within a layer
+ Parameters:
+ d: Recipe-specific datastore
+ destdir: destination directory. Can be the path to the base of the layer or a
+ partial path somewhere within the layer.
+ extrapathhint: a path relative to the base of the layer to try
+ """
+ import bb.cookerdata
+
+ destdir = os.path.abspath(destdir)
+ destlayerdir = find_layerdir(destdir)
+
+ # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
+ confdata = d.createCopy()
+ confdata.setVar('BBFILES', '')
+ confdata.setVar('LAYERDIR', destlayerdir)
+ destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf")
+ confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata)
+ pn = d.getVar('PN', True)
+
+ bbfilespecs = (confdata.getVar('BBFILES', True) or '').split()
+ if destdir == destlayerdir:
+ for bbfilespec in bbfilespecs:
+ if not bbfilespec.endswith('.bbappend'):
+ for match in glob.glob(bbfilespec):
+ splitext = os.path.splitext(os.path.basename(match))
+ if splitext[1] == '.bb':
+ mpn = splitext[0].split('_')[0]
+ if mpn == pn:
+ return os.path.dirname(match)
+
+ # Try to make up a path that matches BBFILES
+ # this is a little crude, but better than nothing
+ bpn = d.getVar('BPN', True)
+ recipefn = os.path.basename(d.getVar('FILE', True))
+ pathoptions = [destdir]
+ if extrapathhint:
+ pathoptions.append(os.path.join(destdir, extrapathhint))
+ if destdir == destlayerdir:
+ pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn))
+ pathoptions.append(os.path.join(destdir, 'recipes', bpn))
+ pathoptions.append(os.path.join(destdir, bpn))
+ elif not destdir.endswith(('/' + pn, '/' + bpn)):
+ pathoptions.append(os.path.join(destdir, bpn))
+ closepath = ''
+ for pathoption in pathoptions:
+ bbfilepath = os.path.join(pathoption, 'test.bb')
+ for bbfilespec in bbfilespecs:
+ if fnmatch.fnmatchcase(bbfilepath, bbfilespec):
+ return pathoption
+ return None
+
def get_bbappend_path(d, destlayerdir, wildcardver=False):
"""Determine how a bbappend for a recipe should be named and located within another layer"""
diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
index 0b305c8..974333f 100644
--- a/meta/lib/oeqa/selftest/devtool.py
+++ b/meta/lib/oeqa/selftest/devtool.py
@@ -5,10 +5,11 @@ import re
import shutil
import tempfile
import glob
+import fnmatch
import oeqa.utils.ftools as ftools
from oeqa.selftest.base import oeSelfTest
-from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer, runqemu
+from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer, runqemu, get_test_layer
from oeqa.utils.decorators import testcase
class DevtoolBase(oeSelfTest):
@@ -1189,3 +1190,157 @@ class DevtoolTests(DevtoolBase):
s = "Microsoft Made No Profit From Anyone's Zunes Yo"
result = runCmd("devtool --quiet selftest-reverse \"%s\"" % s)
self.assertEqual(result.output, s[::-1])
+
+ def _setup_test_devtool_finish_upgrade(self):
+ # Check preconditions
+ self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ self.track_for_cleanup(self.workspacedir)
+ self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+ # Use a "real" recipe from meta-selftest
+ recipe = 'devtool-upgrade-test1'
+ oldversion = '1.5.3'
+ newversion = '1.6.0'
+ oldrecipefile = get_bb_var('FILE', recipe)
+ recipedir = os.path.dirname(oldrecipefile)
+ result = runCmd('git status --porcelain .', cwd=recipedir)
+ if result.output.strip():
+ self.fail('Recipe directory for %s contains uncommitted changes' % recipe)
+ tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+ self.track_for_cleanup(tempdir)
+ # Check that recipe is not already under devtool control
+ result = runCmd('devtool status')
+ self.assertNotIn(recipe, result.output)
+ # Do the upgrade
+ result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, newversion))
+ # Check devtool status and make sure recipe is present
+ result = runCmd('devtool status')
+ self.assertIn(recipe, result.output)
+ self.assertIn(tempdir, result.output)
+ # Make a change to the source
+ result = runCmd('sed -i \'/^#include "pv.h"/a \\/* Here is a new comment *\\/\' src/pv/number.c', cwd=tempdir)
+ result = runCmd('git status --porcelain', cwd=tempdir)
+ self.assertIn('M src/pv/number.c', result.output)
+ result = runCmd('git commit src/pv/number.c -m "Add a comment to the code"', cwd=tempdir)
+ # Check if patch is there
+ recipedir = os.path.dirname(oldrecipefile)
+ olddir = os.path.join(recipedir, recipe + '-' + oldversion)
+ patchfn = '0001-Add-a-note-line-to-the-quick-reference.patch'
+ self.assertTrue(os.path.exists(os.path.join(olddir, patchfn)), 'Original patch file does not exist')
+ return recipe, oldrecipefile, recipedir, olddir, newversion, patchfn
+
+ def test_devtool_finish_upgrade_origlayer(self):
+ recipe, oldrecipefile, recipedir, olddir, newversion, patchfn = self._setup_test_devtool_finish_upgrade()
+ # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things)
+ self.assertIn('/meta-selftest/', recipedir)
+ # Try finish to the original layer
+ self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir))
+ result = runCmd('devtool finish %s meta-selftest' % recipe)
+ result = runCmd('devtool status')
+ self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t')
+ self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish')
+ self.assertFalse(os.path.exists(oldrecipefile), 'Old recipe file should have been deleted but wasn\'t')
+ self.assertFalse(os.path.exists(os.path.join(olddir, patchfn)), 'Old patch file should have been deleted but wasn\'t')
+ newrecipefile = os.path.join(recipedir, '%s_%s.bb' % (recipe, newversion))
+ newdir = os.path.join(recipedir, recipe + '-' + newversion)
+ self.assertTrue(os.path.exists(newrecipefile), 'New recipe file should have been copied into existing layer but wasn\'t')
+ self.assertTrue(os.path.exists(os.path.join(newdir, patchfn)), 'Patch file should have been copied into new directory but wasn\'t')
+ self.assertTrue(os.path.exists(os.path.join(newdir, '0002-Add-a-comment-to-the-code.patch')), 'New patch file should have been created but wasn\'t')
+
+ def test_devtool_finish_upgrade_otherlayer(self):
+ recipe, oldrecipefile, recipedir, olddir, newversion, patchfn = self._setup_test_devtool_finish_upgrade()
+ # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things)
+ self.assertIn('/meta-selftest/', recipedir)
+ # Try finish to a different layer - should create a bbappend
+ # This cleanup isn't strictly necessary but do it anyway just in case it goes wrong and writes to here
+ self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir))
+ oe_core_dir = os.path.join(get_bb_var('COREBASE'), 'meta')
+ newrecipedir = os.path.join(oe_core_dir, 'recipes-test', 'devtool')
+ newrecipefile = os.path.join(newrecipedir, '%s_%s.bb' % (recipe, newversion))
+ self.track_for_cleanup(newrecipedir)
+ result = runCmd('devtool finish %s oe-core' % recipe)
+ result = runCmd('devtool status')
+ self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t')
+ self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish')
+ self.assertTrue(os.path.exists(oldrecipefile), 'Old recipe file should not have been deleted')
+ self.assertTrue(os.path.exists(os.path.join(olddir, patchfn)), 'Old patch file should not have been deleted')
+ newdir = os.path.join(newrecipedir, recipe + '-' + newversion)
+ self.assertTrue(os.path.exists(newrecipefile), 'New recipe file should have been copied into existing layer but wasn\'t')
+ self.assertTrue(os.path.exists(os.path.join(newdir, patchfn)), 'Patch file should have been copied into new directory but wasn\'t')
+ self.assertTrue(os.path.exists(os.path.join(newdir, '0002-Add-a-comment-to-the-code.patch')), 'New patch file should have been created but wasn\'t')
+
+ def _setup_test_devtool_finish_modify(self):
+ # Check preconditions
+ self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ # Try modifying a recipe
+ self.track_for_cleanup(self.workspacedir)
+ recipe = 'mdadm'
+ oldrecipefile = get_bb_var('FILE', recipe)
+ recipedir = os.path.dirname(oldrecipefile)
+ result = runCmd('git status --porcelain .', cwd=recipedir)
+ if result.output.strip():
+ self.fail('Recipe directory for %s contains uncommitted changes' % recipe)
+ tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+ self.track_for_cleanup(tempdir)
+ self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+ result = runCmd('devtool modify %s %s' % (recipe, tempdir))
+ self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found')
+ # Test devtool status
+ result = runCmd('devtool status')
+ self.assertIn(recipe, result.output)
+ self.assertIn(tempdir, result.output)
+ # Make a change to the source
+ result = runCmd('sed -i \'/^#include "mdadm.h"/a \\/* Here is a new comment *\\/\' maps.c', cwd=tempdir)
+ result = runCmd('git status --porcelain', cwd=tempdir)
+ self.assertIn('M maps.c', result.output)
+ result = runCmd('git commit maps.c -m "Add a comment to the code"', cwd=tempdir)
+ for entry in os.listdir(recipedir):
+ filesdir = os.path.join(recipedir, entry)
+ if os.path.isdir(filesdir):
+ break
+ else:
+ self.fail('Unable to find recipe files directory for %s' % recipe)
+ return recipe, oldrecipefile, recipedir, filesdir
+
+ def test_devtool_finish_modify_origlayer(self):
+ recipe, oldrecipefile, recipedir, filesdir = self._setup_test_devtool_finish_modify()
+ # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things)
+ self.assertIn('/meta/', recipedir)
+ # Try finish to the original layer
+ self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir))
+ result = runCmd('devtool finish %s meta' % recipe)
+ result = runCmd('devtool status')
+ self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t')
+ self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish')
+ expected_status = [(' M', '.*/%s$' % os.path.basename(oldrecipefile)),
+ ('??', '.*/.*-Add-a-comment-to-the-code.patch$')]
+ self._check_repo_status(recipedir, expected_status)
+
+ def test_devtool_finish_modify_otherlayer(self):
+ recipe, oldrecipefile, recipedir, filesdir = self._setup_test_devtool_finish_modify()
+ # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things)
+ self.assertIn('/meta/', recipedir)
+ relpth = os.path.relpath(recipedir, os.path.join(get_bb_var('COREBASE'), 'meta'))
+ appenddir = os.path.join(get_test_layer(), relpth)
+ self.track_for_cleanup(appenddir)
+ # Try finish to the original layer
+ self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir))
+ result = runCmd('devtool finish %s meta-selftest' % recipe)
+ result = runCmd('devtool status')
+ self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t')
+ self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish')
+ result = runCmd('git status --porcelain .', cwd=recipedir)
+ if result.output.strip():
+ self.fail('Recipe directory for %s contains the following unexpected changes after finish:\n%s' % (recipe, result.output.strip()))
+ appendfile = os.path.join(appenddir, os.path.splitext(os.path.basename(oldrecipefile))[0] + '.bbappend')
+ self.assertTrue(os.path.exists(appendfile), 'bbappend %s should have been created but wasn\'t' % appendfile)
+ newdir = os.path.join(appenddir, recipe)
+ files = os.listdir(newdir)
+ foundpatch = None
+ for fn in files:
+ if fnmatch.fnmatch(fn, '*-Add-a-comment-to-the-code.patch'):
+ foundpatch = fn
+ if not foundpatch:
+ self.fail('No patch file created next to bbappend')
+ files.remove(foundpatch)
+ if files:
+ self.fail('Unexpected file(s) copied next to bbappend: %s' % ', '.join(files))
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 5a5995f..9c09533 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -1,6 +1,6 @@
# Development tool - standard commands plugin
#
-# Copyright (C) 2014-2015 Intel Corporation
+# Copyright (C) 2014-2016 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -1394,6 +1394,106 @@ def reset(args, config, basepath, workspace):
return 0
+def _get_layer(layername, d):
+ """Determine the base layer path for the specified layer name/path"""
+ layerdirs = d.getVar('BBLAYERS', True).split()
+ layers = {os.path.basename(p): p for p in layerdirs}
+ # Provide some shortcuts
+ if layername.lower() in ['oe-core', 'openembedded-core']:
+ layerdir = layers.get('meta', None)
+ else:
+ layerdir = layers.get(layername, None)
+ if layerdir:
+ layerdir = os.path.abspath(layerdir)
+ return layerdir or layername
+
+def finish(args, config, basepath, workspace):
+ """Entry point for the devtool 'finish' subcommand"""
+ import bb
+ import oe.recipeutils
+
+ check_workspace_recipe(workspace, args.recipename)
+
+ tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
+ try:
+ rd = parse_recipe(config, tinfoil, args.recipename, True)
+ if not rd:
+ return 1
+
+ destlayerdir = _get_layer(args.destination, tinfoil.config_data)
+ origlayerdir = oe.recipeutils.find_layerdir(rd.getVar('FILE', True))
+
+ if not os.path.isdir(destlayerdir):
+ raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination)
+
+ if os.path.abspath(destlayerdir) == config.workspace_path:
+ raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination)
+
+ # If it's an upgrade, grab the original path
+ origpath = None
+ origfilelist = None
+ append = workspace[args.recipename]['bbappend']
+ with open(append, 'r') as f:
+ for line in f:
+ if line.startswith('# original_path:'):
+ origpath = line.split(':')[1].strip()
+ elif line.startswith('# original_files:'):
+ origfilelist = line.split(':')[1].split()
+
+ if origlayerdir == config.workspace_path:
+ # Recipe file itself is in workspace, update it there first
+ appendlayerdir = None
+ origrelpath = None
+ if origpath:
+ origlayerpath = oe.recipeutils.find_layerdir(origpath)
+ if origlayerpath:
+ origrelpath = os.path.relpath(origpath, origlayerpath)
+ destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath)
+ if not destpath:
+ raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir))
+ elif destlayerdir == origlayerdir:
+ # Same layer, update the original recipe
+ appendlayerdir = None
+ destpath = None
+ else:
+ # Create/update a bbappend in the specified layer
+ appendlayerdir = destlayerdir
+ destpath = None
+
+ # Remove any old files in the case of an upgrade
+ if origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == oe.recipeutils.find_layerdir(destlayerdir):
+ for fn in origfilelist:
+ fnp = os.path.join(origpath, fn)
+ try:
+ os.remove(fnp)
+ except FileNotFoundError:
+ pass
+
+ # Actually update the recipe / bbappend
+ _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, initial_rev=args.initial_rev)
+
+ if origlayerdir == config.workspace_path and destpath:
+ # Recipe file itself is in the workspace - need to move it and any
+ # associated files to the specified layer
+ logger.info('Moving recipe file to %s' % destpath)
+ recipedir = os.path.dirname(rd.getVar('FILE', True))
+ for root, _, files in os.walk(recipedir):
+ for fn in files:
+ srcpath = os.path.join(root, fn)
+ relpth = os.path.relpath(os.path.dirname(srcpath), recipedir)
+ destdir = os.path.abspath(os.path.join(destpath, relpth))
+ bb.utils.mkdirhier(destdir)
+ shutil.move(srcpath, os.path.join(destdir, fn))
+
+ finally:
+ tinfoil.shutdown()
+
+ # Everything else has succeeded, we can now reset
+ _reset([args.recipename], no_clean=False, config=config, basepath=basepath, workspace=workspace)
+
+ return 0
+
+
def get_default_srctree(config, recipename=''):
"""Get the default srctree path"""
srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path)
@@ -1481,3 +1581,12 @@ def register_commands(subparsers, context):
parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)')
parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output')
parser_reset.set_defaults(func=reset)
+
+ parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace',
+ description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified.',
+ group='working', order=-100)
+ parser_finish.add_argument('recipename', help='Recipe to finish')
+ parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.')
+ parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
+ parser_finish.add_argument('--initial-rev', help='Override starting revision for patches')
+ parser_finish.set_defaults(func=finish)
--
2.5.5
^ permalink raw reply related [flat|nested] 8+ messages in thread