* [PATCH v2 2/3] devtool: finish: handle patching when S points to subdir of a git repo
2022-07-13 1:41 [PATCH v2 0/3] devtool fixes Paul Eggleton
2022-07-13 1:41 ` [PATCH v2 1/3] patch: handle if S points to a subdirectory of a git repo Paul Eggleton
@ 2022-07-13 1:41 ` Paul Eggleton
2022-07-13 1:41 ` [PATCH v2 3/3] oe-selftest: devtool: test modify git recipe building from a subdir Paul Eggleton
2 siblings, 0 replies; 5+ messages in thread
From: Paul Eggleton @ 2022-07-13 1:41 UTC (permalink / raw)
To: openembedded-core
From: Paul Eggleton <paul.eggleton@microsoft.com>
If devtool finish needs to create a patch and have it applied to the
sources for a recipe where S points to a subdirectory of the sources,
then the patch needs to be applied at the root of the repo i.e. we need
to add a patchdir= parameter to the SRC_URI entry.
Signed-off-by: Paul Eggleton <paul.eggleton@microsoft.com>
---
meta/lib/oe/recipeutils.py | 9 +++++++--
scripts/lib/devtool/standard.py | 25 +++++++++++++++++++------
2 files changed, 26 insertions(+), 8 deletions(-)
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index 872ff97..b04992c 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -666,7 +666,7 @@ def get_bbappend_path(d, destlayerdir, wildcardver=False):
return (appendpath, pathok)
-def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None):
+def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None, params=None):
"""
Writes a bbappend file for a recipe
Parameters:
@@ -696,6 +696,9 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
redirect_output:
If specified, redirects writing the output file to the
specified directory (for dry-run purposes)
+ params:
+ Parameters to use when adding entries to SRC_URI. If specified,
+ should be a list of dicts with the same length as srcfiles.
"""
if not removevalues:
@@ -762,12 +765,14 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
copyfiles = {}
if srcfiles:
instfunclines = []
- for newfile, origsrcfile in srcfiles.items():
+ for i, (newfile, origsrcfile) in enumerate(srcfiles.items()):
srcfile = origsrcfile
srcurientry = None
if not srcfile:
srcfile = os.path.basename(newfile)
srcurientry = 'file://%s' % srcfile
+ if params and params[i]:
+ srcurientry = '%s;%s' % (srcurientry, ';'.join('%s=%s' % (k,v) for k,v in params[i].items()))
# Double-check it's not there already
# FIXME do we care if the entry is added by another bbappend that might go away?
if not srcurientry in rd.getVar('SRC_URI').split():
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index e53569c..c98bfe8 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -1606,6 +1606,19 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
if not os.path.exists(append):
raise DevtoolError('unable to find workspace bbappend for recipe %s' %
recipename)
+ srctreebase = workspace[recipename]['srctreebase']
+ relpatchdir = os.path.relpath(srctreebase, srctree)
+ if relpatchdir == '.':
+ patchdir_params = {}
+ else:
+ patchdir_params = {'patchdir': relpatchdir}
+
+ def srcuri_entry(fname):
+ if patchdir_params:
+ paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items())
+ else:
+ paramstr = ''
+ return 'file://%s%s' % (basepath, paramstr)
initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh)
if not initial_rev:
@@ -1627,7 +1640,6 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
new_f = {}
del_f = {}
else:
- srctreebase = workspace[recipename]['srctreebase']
upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
remove_files = []
@@ -1663,14 +1675,15 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
removedentries, remaining = _remove_file_entries(
srcuri, remove_files)
if removedentries or remaining:
- remaining = ['file://' + os.path.basename(item) for
+ remaining = [srcuri_entry(os.path.basename(item)) for
item in remaining]
removevalues = {'SRC_URI': removedentries + remaining}
appendfile, destpath = oe.recipeutils.bbappend_recipe(
rd, appendlayerdir, files,
wildcardver=wildcard_version,
removevalues=removevalues,
- redirect_output=dry_run_outdir)
+ redirect_output=dry_run_outdir,
+ params=[patchdir_params] * len(files))
else:
logger.info('No patches or local source files needed updating')
else:
@@ -1694,7 +1707,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
# replace the entry in SRC_URI with our local version
logger.info('Replacing remote patch %s with updated local version' % basepath)
path = os.path.join(files_dir, basepath)
- _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath)
+ _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath))
updaterecipe = True
else:
logger.info('Updating patch %s%s' % (basepath, dry_run_suffix))
@@ -1708,7 +1721,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
os.path.join(files_dir, basepath),
dry_run_outdir=dry_run_outdir,
base_outdir=recipedir)
- srcuri.append('file://%s' % basepath)
+ srcuri.append(srcuri_entry(basepath))
updaterecipe = True
for basepath, path in new_p.items():
logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix))
@@ -1716,7 +1729,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
os.path.join(files_dir, basepath),
dry_run_outdir=dry_run_outdir,
base_outdir=recipedir)
- srcuri.append('file://%s' % basepath)
+ srcuri.append(srcuri_entry(basepath))
updaterecipe = True
# Update recipe, if needed
if _remove_file_entries(srcuri, remove_files)[0]:
--
1.8.3.1
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH v2 3/3] oe-selftest: devtool: test modify git recipe building from a subdir
2022-07-13 1:41 [PATCH v2 0/3] devtool fixes Paul Eggleton
2022-07-13 1:41 ` [PATCH v2 1/3] patch: handle if S points to a subdirectory of a git repo Paul Eggleton
2022-07-13 1:41 ` [PATCH v2 2/3] devtool: finish: handle patching when S points to subdir " Paul Eggleton
@ 2022-07-13 1:41 ` Paul Eggleton
2 siblings, 0 replies; 5+ messages in thread
From: Paul Eggleton @ 2022-07-13 1:41 UTC (permalink / raw)
To: openembedded-core
From: Paul Eggleton <paul.eggleton@microsoft.com>
Add a test that verifies that devtool modify + devtool finish do the
right thing on a recipe that fetches from git and sets S to point to
a subdirectory of the source tree. We have a few examples among the core
recipes, dos2unix is a convenient one so let's use that. (The test first
verifies that that is still true in case the recipe is changed in
future.)
Signed-off-by: Paul Eggleton <paul.eggleton@microsoft.com>
---
meta/lib/oeqa/selftest/cases/devtool.py | 114 +++++++++++++++++++++++++++-----
1 file changed, 97 insertions(+), 17 deletions(-)
diff --git a/meta/lib/oeqa/selftest/cases/devtool.py b/meta/lib/oeqa/selftest/cases/devtool.py
index ddf6c0c..34fc791 100644
--- a/meta/lib/oeqa/selftest/cases/devtool.py
+++ b/meta/lib/oeqa/selftest/cases/devtool.py
@@ -218,6 +218,34 @@ class DevtoolTestCase(OESelftestTestCase):
filelist.append(' '.join(splitline))
return filelist
+ def _check_diff(self, diffoutput, addlines, removelines):
+ """Check output from 'git diff' matches expectation"""
+ remaining_addlines = addlines[:]
+ remaining_removelines = removelines[:]
+ for line in diffoutput.splitlines():
+ if line.startswith('+++') or line.startswith('---'):
+ continue
+ elif line.startswith('+'):
+ matched = False
+ for item in addlines:
+ if re.match(item, line[1:].strip()):
+ matched = True
+ remaining_addlines.remove(item)
+ break
+ self.assertTrue(matched, 'Unexpected diff add line: %s' % line)
+ elif line.startswith('-'):
+ matched = False
+ for item in removelines:
+ if re.match(item, line[1:].strip()):
+ matched = True
+ remaining_removelines.remove(item)
+ break
+ self.assertTrue(matched, 'Unexpected diff remove line: %s' % line)
+ if remaining_addlines:
+ self.fail('Expected added lines not found: %s' % remaining_addlines)
+ if remaining_removelines:
+ self.fail('Expected removed lines not found: %s' % remaining_removelines)
+
class DevtoolBase(DevtoolTestCase):
@@ -718,6 +746,7 @@ class DevtoolModifyTests(DevtoolBase):
self.assertTrue(bbclassextended, 'None of these recipes are BBCLASSEXTENDed to native - need to adjust testrecipes list: %s' % ', '.join(testrecipes))
self.assertTrue(inheritnative, 'None of these recipes do "inherit native" - need to adjust testrecipes list: %s' % ', '.join(testrecipes))
+
def test_devtool_modify_localfiles_only(self):
# Check preconditions
testrecipe = 'base-files'
@@ -930,23 +959,7 @@ class DevtoolUpdateTests(DevtoolBase):
srcurilines[0] = 'SRC_URI = "' + srcurilines[0]
srcurilines.append('"')
removelines = ['SRCREV = ".*"'] + srcurilines
- for line in result.output.splitlines():
- if line.startswith('+++') or line.startswith('---'):
- continue
- elif line.startswith('+'):
- matched = False
- for item in addlines:
- if re.match(item, line[1:].strip()):
- matched = True
- break
- self.assertTrue(matched, 'Unexpected diff add line: %s' % line)
- elif line.startswith('-'):
- matched = False
- for item in removelines:
- if re.match(item, line[1:].strip()):
- matched = True
- break
- self.assertTrue(matched, 'Unexpected diff remove line: %s' % line)
+ self._check_diff(result.output, addlines, removelines)
# Now try with auto mode
runCmd('cd %s; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, os.path.basename(recipefile)))
result = runCmd('devtool update-recipe %s' % testrecipe)
@@ -1316,6 +1329,73 @@ class DevtoolUpdateTests(DevtoolBase):
expected_status = []
self._check_repo_status(os.path.dirname(recipefile), expected_status)
+ def test_devtool_finish_modify_git_subdir(self):
+ # Check preconditions
+ testrecipe = 'dos2unix'
+ bb_vars = get_bb_vars(['SRC_URI', 'S', 'WORKDIR', 'FILE'], testrecipe)
+ self.assertIn('git://', bb_vars['SRC_URI'], 'This test expects the %s recipe to be a git recipe' % testrecipe)
+ workdir_git = '%s/git/' % bb_vars['WORKDIR']
+ if not bb_vars['S'].startswith(workdir_git):
+ self.fail('This test expects the %s recipe to be building from a subdirectory of the git repo' % testrecipe)
+ subdir = bb_vars['S'].split(workdir_git, 1)[1]
+ # Clean up anything in the workdir/sysroot/sstate cache
+ bitbake('%s -c cleansstate' % testrecipe)
+ # Try modifying a recipe
+ tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+ self.track_for_cleanup(tempdir)
+ self.track_for_cleanup(self.workspacedir)
+ self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)
+ self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+ result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
+ testsrcfile = os.path.join(tempdir, subdir, 'dos2unix.c')
+ self.assertExists(testsrcfile, 'Extracted source could not be found')
+ self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. devtool output: %s' % result.output)
+ self.assertNotExists(os.path.join(tempdir, subdir, '.git'), 'Subdirectory has been initialised as a git repo')
+ # Check git repo
+ self._check_src_repo(tempdir)
+ # Modify file
+ runCmd("sed -i '1s:^:/* Add a comment */\\n:' %s" % testsrcfile)
+ result = runCmd('git commit -a -m "Add a comment"', cwd=tempdir)
+ # Now try updating original recipe
+ recipefile = bb_vars['FILE']
+ recipedir = os.path.dirname(recipefile)
+ self.add_command_to_tearDown('cd %s; rm -f %s/*.patch; git checkout .' % (recipedir, testrecipe))
+ result = runCmd('devtool update-recipe %s' % testrecipe)
+ expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)),
+ ('??', '.*/%s/%s/$' % (testrecipe, testrecipe))]
+ self._check_repo_status(os.path.dirname(recipefile), expected_status)
+ result = runCmd('git diff %s' % os.path.basename(recipefile), cwd=os.path.dirname(recipefile))
+ removelines = ['SRC_URI = "git://.*"']
+ addlines = [
+ 'SRC_URI = "git://.* \\\\',
+ 'file://0001-Add-a-comment.patch;patchdir=.. \\\\',
+ '"'
+ ]
+ self._check_diff(result.output, addlines, removelines)
+ # Put things back so we can run devtool finish on a different layer
+ runCmd('cd %s; rm -f %s/*.patch; git checkout .' % (recipedir, testrecipe))
+ # Run devtool finish
+ res = re.search('recipes-.*', recipedir)
+ self.assertTrue(res, 'Unable to find recipe subdirectory')
+ recipesubdir = res[0]
+ self.add_command_to_tearDown('rm -rf %s' % os.path.join(self.testlayer_path, recipesubdir))
+ result = runCmd('devtool finish %s meta-selftest' % testrecipe)
+ # Check bbappend file contents
+ appendfn = os.path.join(self.testlayer_path, recipesubdir, '%s_%%.bbappend' % testrecipe)
+ with open(appendfn, 'r') as f:
+ appendlines = f.readlines()
+ expected_appendlines = [
+ 'FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n',
+ '\n',
+ 'SRC_URI += "file://0001-Add-a-comment.patch;patchdir=.."\n',
+ '\n'
+ ]
+ self.assertEqual(appendlines, expected_appendlines)
+ self.assertExists(os.path.join(os.path.dirname(appendfn), testrecipe, '0001-Add-a-comment.patch'))
+ # Try building
+ bitbake('%s -c patch' % testrecipe)
+
+
class DevtoolExtractTests(DevtoolBase):
def test_devtool_extract(self):
--
1.8.3.1
^ permalink raw reply related [flat|nested] 5+ messages in thread