* [PATCH v2 1/7] recipeutils: implement get_recipe_local_files()
2015-06-03 14:50 [PATCH v2 0/7] devtool: improve handling of local source files Markus Lehtonen
@ 2015-06-03 14:50 ` Markus Lehtonen
2015-06-03 14:50 ` [PATCH v2 2/7] oe.patch.GitApplyTree: add paths argument to extractPatches Markus Lehtonen
` (6 subsequent siblings)
7 siblings, 0 replies; 10+ messages in thread
From: Markus Lehtonen @ 2015-06-03 14:50 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oe/recipeutils.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index f05b6c0..4bd5ce7 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -278,6 +278,14 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True):
return remotes
+def get_recipe_local_files(d):
+ """Get a list of local files in SRC_URI within a recipe."""
+ uris = (d.getVar('SRC_URI', True) or "").split()
+ fetch = bb.fetch2.Fetch(uris, d)
+ return dict([(fetch.ud[uri].basepath, fetch.localpath(uri)) for uri in uris
+ if uri.startswith('file://')])
+
+
def get_recipe_patches(d):
"""Get a list of the patches included in SRC_URI within a recipe."""
patchfiles = []
--
2.1.4
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 2/7] oe.patch.GitApplyTree: add paths argument to extractPatches
2015-06-03 14:50 [PATCH v2 0/7] devtool: improve handling of local source files Markus Lehtonen
2015-06-03 14:50 ` [PATCH v2 1/7] recipeutils: implement get_recipe_local_files() Markus Lehtonen
@ 2015-06-03 14:50 ` Markus Lehtonen
2015-06-03 14:50 ` [PATCH v2 3/7] oe-selftest: devtool: add method for checking workspace dir Markus Lehtonen
` (5 subsequent siblings)
7 siblings, 0 replies; 10+ messages in thread
From: Markus Lehtonen @ 2015-06-03 14:50 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
Makes it possible to define which paths are included in the patches.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oe/patch.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/meta/lib/oe/patch.py b/meta/lib/oe/patch.py
index afb0013..6da2511 100644
--- a/meta/lib/oe/patch.py
+++ b/meta/lib/oe/patch.py
@@ -337,12 +337,15 @@ class GitApplyTree(PatchTree):
return (tmpfile, cmd)
@staticmethod
- def extractPatches(tree, startcommit, outdir):
+ def extractPatches(tree, startcommit, outdir, paths=None):
import tempfile
import shutil
tempdir = tempfile.mkdtemp(prefix='oepatch')
try:
shellcmd = ["git", "format-patch", startcommit, "-o", tempdir]
+ if paths:
+ shellcmd.append('--')
+ shellcmd.extend(paths)
out = runcmd(["sh", "-c", " ".join(shellcmd)], tree)
if out:
for srcfile in out.split():
--
2.1.4
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 3/7] oe-selftest: devtool: add method for checking workspace dir
2015-06-03 14:50 [PATCH v2 0/7] devtool: improve handling of local source files Markus Lehtonen
2015-06-03 14:50 ` [PATCH v2 1/7] recipeutils: implement get_recipe_local_files() Markus Lehtonen
2015-06-03 14:50 ` [PATCH v2 2/7] oe.patch.GitApplyTree: add paths argument to extractPatches Markus Lehtonen
@ 2015-06-03 14:50 ` Markus Lehtonen
2015-06-03 14:50 ` [PATCH v2 4/7] oe-selftest: devtool: add method for checking srctree repo Markus Lehtonen
` (4 subsequent siblings)
7 siblings, 0 replies; 10+ messages in thread
From: Markus Lehtonen @ 2015-06-03 14:50 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
In order to remove some code duplication.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/selftest/devtool.py | 64 +++++++++++++++------------------------
1 file changed, 24 insertions(+), 40 deletions(-)
diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
index 4e22e1d..c01e361 100644
--- a/meta/lib/oeqa/selftest/devtool.py
+++ b/meta/lib/oeqa/selftest/devtool.py
@@ -63,10 +63,17 @@ class DevtoolBase(oeSelfTest):
class DevtoolTests(DevtoolBase):
+ def _get_workspace_dir(self):
+ """Get workspace directory"""
+ workspacedir = os.path.join(self.builddir, 'workspace')
+ self.assertTrue(not os.path.exists(workspacedir),
+ 'This test cannot be run with a workspace directory '
+ 'under the build directory')
+ return workspacedir
+
def test_create_workspace(self):
# Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
result = runCmd('bitbake-layers show-layers')
self.assertTrue('/workspace' not in result.output, 'This test cannot be run with a workspace layer in bblayers.conf')
# Try creating a workspace layer with a specific path
@@ -86,9 +93,7 @@ class DevtoolTests(DevtoolBase):
self.assertIn(workspacedir, result.output)
def test_devtool_add(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
# Fetch source
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
self.track_for_cleanup(tempdir)
@@ -120,9 +125,7 @@ class DevtoolTests(DevtoolBase):
self.assertTrue(os.path.isfile(os.path.join(installdir, bindir, 'pv')), 'pv binary not found in D')
def test_devtool_add_library(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
# We don't have the ability to pick up this dependency automatically yet...
bitbake('libusb1')
# Fetch source
@@ -160,9 +163,7 @@ class DevtoolTests(DevtoolBase):
self.assertFalse(os.path.isfile(os.path.join(staging_libdir, 'libftdi1.so.2.1.0')), 'libftdi binary still found in STAGING_LIBDIR after cleaning')
def test_devtool_add_fetch(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
# Fetch source
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
self.track_for_cleanup(tempdir)
@@ -206,9 +207,7 @@ class DevtoolTests(DevtoolBase):
self._test_recipe_contents(recipefile, checkvars, [])
def test_devtool_add_fetch_git(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
# Fetch source
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
self.track_for_cleanup(tempdir)
@@ -257,9 +256,7 @@ class DevtoolTests(DevtoolBase):
self._test_recipe_contents(recipefile, checkvars, [])
def test_devtool_modify(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
# Clean up anything in the workdir/sysroot/sstate cache
bitbake('mdadm -c cleansstate')
# Try modifying a recipe
@@ -308,9 +305,7 @@ class DevtoolTests(DevtoolBase):
self.assertFalse(matches, 'Stamp files exist for recipe mdadm that should have been cleaned')
def test_devtool_modify_invalid(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
# Try modifying some recipes
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
self.track_for_cleanup(tempdir)
@@ -341,8 +336,7 @@ class DevtoolTests(DevtoolBase):
def test_devtool_modify_git(self):
# Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
testrecipe = 'mkelfimage'
src_uri = get_bb_var('SRC_URI', testrecipe)
self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)
@@ -374,8 +368,7 @@ class DevtoolTests(DevtoolBase):
def test_devtool_modify_localfiles(self):
# Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
testrecipe = 'lighttpd'
src_uri = (get_bb_var('SRC_URI', testrecipe) or '').split()
foundlocal = False
@@ -406,8 +399,7 @@ class DevtoolTests(DevtoolBase):
def test_devtool_update_recipe(self):
# Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
testrecipe = 'minicom'
recipefile = get_bb_var('FILE', testrecipe)
src_uri = get_bb_var('SRC_URI', testrecipe)
@@ -452,8 +444,7 @@ class DevtoolTests(DevtoolBase):
def test_devtool_update_recipe_git(self):
# Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
testrecipe = 'mtd-utils'
recipefile = get_bb_var('FILE', testrecipe)
src_uri = get_bb_var('SRC_URI', testrecipe)
@@ -526,8 +517,7 @@ class DevtoolTests(DevtoolBase):
def test_devtool_update_recipe_append(self):
# Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
testrecipe = 'mdadm'
recipefile = get_bb_var('FILE', testrecipe)
src_uri = get_bb_var('SRC_URI', testrecipe)
@@ -601,8 +591,7 @@ class DevtoolTests(DevtoolBase):
def test_devtool_update_recipe_append_git(self):
# Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
testrecipe = 'mtd-utils'
recipefile = get_bb_var('FILE', testrecipe)
src_uri = get_bb_var('SRC_URI', testrecipe)
@@ -696,9 +685,7 @@ class DevtoolTests(DevtoolBase):
# Deleting isn't expected to work under these circumstances
def test_devtool_extract(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
# Try devtool extract
self.track_for_cleanup(tempdir)
@@ -709,9 +696,7 @@ class DevtoolTests(DevtoolBase):
self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')
def test_devtool_reset_all(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
self.track_for_cleanup(tempdir)
self.track_for_cleanup(workspacedir)
@@ -760,8 +745,7 @@ class DevtoolTests(DevtoolBase):
break
else:
self.skipTest('No tap devices found - you must set up tap devices with scripts/runqemu-gen-tapdevs before running this test')
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
import pexpect
# Definitions
testrecipe = 'mdadm'
--
2.1.4
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 4/7] oe-selftest: devtool: add method for checking srctree repo
2015-06-03 14:50 [PATCH v2 0/7] devtool: improve handling of local source files Markus Lehtonen
` (2 preceding siblings ...)
2015-06-03 14:50 ` [PATCH v2 3/7] oe-selftest: devtool: add method for checking workspace dir Markus Lehtonen
@ 2015-06-03 14:50 ` Markus Lehtonen
2015-06-03 14:50 ` [PATCH v2 5/7] devtool: update-recipe: update local files directly Markus Lehtonen
` (3 subsequent siblings)
7 siblings, 0 replies; 10+ messages in thread
From: Markus Lehtonen @ 2015-06-03 14:50 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
Removes some code duplication.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/selftest/devtool.py | 50 +++++++++++++++------------------------
1 file changed, 19 insertions(+), 31 deletions(-)
diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
index c01e361..08ed2eb 100644
--- a/meta/lib/oeqa/selftest/devtool.py
+++ b/meta/lib/oeqa/selftest/devtool.py
@@ -71,6 +71,18 @@ class DevtoolTests(DevtoolBase):
'under the build directory')
return workspacedir
+ def _check_src_repo(self, repo_dir):
+ """Check srctree git repository"""
+ self.assertTrue(os.path.isdir(os.path.join(repo_dir, '.git')),
+ 'git repository for external source tree not found')
+ result = runCmd('git status --porcelain', cwd=repo_dir)
+ self.assertEqual(result.output.strip(), "",
+ 'Created git repo is not clean')
+ result = runCmd('git symbolic-ref HEAD', cwd=repo_dir)
+ self.assertEqual(result.output.strip(), "refs/heads/devtool",
+ 'Wrong branch in git repo')
+
+
def test_create_workspace(self):
# Check preconditions
workspacedir = self._get_workspace_dir()
@@ -267,7 +279,6 @@ class DevtoolTests(DevtoolBase):
self.add_command_to_tearDown('bitbake -c clean mdadm')
result = runCmd('devtool modify mdadm -x %s' % tempdir)
self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found')
- self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')
self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')
matches = glob.glob(os.path.join(workspacedir, 'appends', 'mdadm_*.bbappend'))
self.assertTrue(matches, 'bbappend not created')
@@ -276,10 +287,7 @@ class DevtoolTests(DevtoolBase):
self.assertIn('mdadm', result.output)
self.assertIn(tempdir, result.output)
# Check git repo
- result = runCmd('git status --porcelain', cwd=tempdir)
- self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')
- result = runCmd('git symbolic-ref HEAD', cwd=tempdir)
- self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')
+ self._check_src_repo(tempdir)
# Try building
bitbake('mdadm')
# Try making (minor) modifications to the source
@@ -350,7 +358,6 @@ class DevtoolTests(DevtoolBase):
self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)
result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found')
- self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')
self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')
matches = glob.glob(os.path.join(workspacedir, 'appends', 'mkelfimage_*.bbappend'))
self.assertTrue(matches, 'bbappend not created')
@@ -359,10 +366,7 @@ class DevtoolTests(DevtoolBase):
self.assertIn(testrecipe, result.output)
self.assertIn(tempdir, result.output)
# Check git repo
- result = runCmd('git status --porcelain', cwd=tempdir)
- self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')
- result = runCmd('git symbolic-ref HEAD', cwd=tempdir)
- self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')
+ self._check_src_repo(tempdir)
# Try building
bitbake(testrecipe)
@@ -414,11 +418,7 @@ class DevtoolTests(DevtoolBase):
# (don't bother with cleaning the recipe on teardown, we won't be building it)
result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
# Check git repo
- self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')
- result = runCmd('git status --porcelain', cwd=tempdir)
- self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')
- result = runCmd('git symbolic-ref HEAD', cwd=tempdir)
- self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')
+ self._check_src_repo(tempdir)
# Add a couple of commits
# FIXME: this only tests adding, need to also test update and remove
result = runCmd('echo "Additional line" >> README', cwd=tempdir)
@@ -464,11 +464,7 @@ class DevtoolTests(DevtoolBase):
# (don't bother with cleaning the recipe on teardown, we won't be building it)
result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
# Check git repo
- self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')
- result = runCmd('git status --porcelain', cwd=tempdir)
- self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')
- result = runCmd('git symbolic-ref HEAD', cwd=tempdir)
- self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')
+ self._check_src_repo(tempdir)
# Add a couple of commits
# FIXME: this only tests adding, need to also test update and remove
result = runCmd('echo "# Additional line" >> Makefile', cwd=tempdir)
@@ -534,11 +530,7 @@ class DevtoolTests(DevtoolBase):
# (don't bother with cleaning the recipe on teardown, we won't be building it)
result = runCmd('devtool modify %s -x %s' % (testrecipe, tempsrcdir))
# Check git repo
- self.assertTrue(os.path.isdir(os.path.join(tempsrcdir, '.git')), 'git repository for external source tree not found')
- result = runCmd('git status --porcelain', cwd=tempsrcdir)
- self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')
- result = runCmd('git symbolic-ref HEAD', cwd=tempsrcdir)
- self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')
+ self._check_src_repo(tempdir)
# Add a commit
result = runCmd("sed 's!\\(#define VERSION\\W*\"[^\"]*\\)\"!\\1-custom\"!' -i ReadMe.c", cwd=tempsrcdir)
result = runCmd('git commit -a -m "Add our custom version"', cwd=tempsrcdir)
@@ -612,11 +604,7 @@ class DevtoolTests(DevtoolBase):
# (don't bother with cleaning the recipe on teardown, we won't be building it)
result = runCmd('devtool modify %s -x %s' % (testrecipe, tempsrcdir))
# Check git repo
- self.assertTrue(os.path.isdir(os.path.join(tempsrcdir, '.git')), 'git repository for external source tree not found')
- result = runCmd('git status --porcelain', cwd=tempsrcdir)
- self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')
- result = runCmd('git symbolic-ref HEAD', cwd=tempsrcdir)
- self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')
+ self._check_src_repo(tempdir)
# Add a commit
result = runCmd('echo "# Additional line" >> Makefile', cwd=tempsrcdir)
result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir)
@@ -693,7 +681,7 @@ class DevtoolTests(DevtoolBase):
self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
result = runCmd('devtool extract remake %s' % tempdir)
self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found')
- self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')
+ self._check_src_repo(tempdir)
def test_devtool_reset_all(self):
workspacedir = self._get_workspace_dir()
--
2.1.4
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 5/7] devtool: update-recipe: update local files directly
2015-06-03 14:50 [PATCH v2 0/7] devtool: improve handling of local source files Markus Lehtonen
` (3 preceding siblings ...)
2015-06-03 14:50 ` [PATCH v2 4/7] oe-selftest: devtool: add method for checking srctree repo Markus Lehtonen
@ 2015-06-03 14:50 ` Markus Lehtonen
2015-06-03 14:50 ` [PATCH v2 6/7] devtool: extract: always import local files to srctree Markus Lehtonen
` (2 subsequent siblings)
7 siblings, 0 replies; 10+ messages in thread
From: Markus Lehtonen @ 2015-06-03 14:50 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
Non-remote source files (i.e. SRC_URI files that use the file:// URI
prefix and thus reside in the "recipe space" in the local copy of the
metadata) are imported into srctree (with devtool extract) in the case
S=WORKDIR. If these files are local (i.e. they reside in the "recipe
space" and not behind a remote URL) we don't want create a patch against
them, but, rather copy our modified version over the original source.
NOTE: if new files are created, they are represented as patches, rather
than copied over the orignal source.
[YOCTO #7602]
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/selftest/devtool.py | 44 +++++++++++++++++++++++++++++++++++
scripts/lib/devtool/standard.py | 48 ++++++++++++++++++++++++++++++++++++---
2 files changed, 89 insertions(+), 3 deletions(-)
diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
index 08ed2eb..6aaf5a5 100644
--- a/meta/lib/oeqa/selftest/devtool.py
+++ b/meta/lib/oeqa/selftest/devtool.py
@@ -672,6 +672,50 @@ class DevtoolTests(DevtoolBase):
self.assertEqual(expectedlines, f.readlines())
# Deleting isn't expected to work under these circumstances
+ def test_devtool_update_recipe_local_files(self):
+ """Check that local source files are copied over instead of patched"""
+ workspacedir = self._get_workspace_dir()
+ testrecipe = 'makedevs'
+ recipefile = get_bb_var('FILE', testrecipe)
+ # Setup srctree for modifying the recipe
+ tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+ self.track_for_cleanup(tempdir)
+ self.track_for_cleanup(workspacedir)
+ self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+ # (don't bother with cleaning the recipe on teardown, we won't be
+ # building it)
+ result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
+ # Check git repo
+ self._check_src_repo(tempdir)
+ # Edit / commit local source
+ runCmd('echo "/* New comment */" >> makedevs.c', cwd=tempdir)
+ runCmd('git commit -am "My change"', cwd=tempdir)
+ runCmd('echo "Foo" > new-file', cwd=tempdir)
+ runCmd('git add new-file', cwd=tempdir)
+ runCmd('git commit -m "Add new file"', cwd=tempdir)
+ self.add_command_to_tearDown('cd %s; git clean -fd .; git checkout .' %
+ os.path.dirname(recipefile))
+ runCmd('devtool update-recipe %s' % testrecipe)
+ result = runCmd('git status . --porcelain',
+ cwd=os.path.dirname(recipefile))
+ status = result.output.splitlines()
+ self.assertEqual(len(status), 3,
+ 'Less/more files modified than expected. '
+ 'Entire status:\n%s' % result.output)
+ for line in status:
+ if line.endswith('makedevs.c'):
+ self.assertEqual(line[:3], ' M ',
+ 'Unexpected status in line: %s' % line)
+ elif line.endswith('0001-Add-new-file.patch'):
+ self.assertEqual(line[:3], '?? ',
+ 'Unexpected status in line: %s' % line)
+ elif re.search('%s_[^_]*.bb$' % testrecipe, line):
+ self.assertEqual(line[:3], ' M ',
+ 'Unexpected status in line: %s' % line)
+ else:
+ raise AssertionError('Unexpected modified file in status: %s' %
+ line)
+
def test_devtool_extract(self):
workspacedir = self._get_workspace_dir()
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index c5b32d8..75cc495 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -187,6 +187,16 @@ def _parse_recipe(config, tinfoil, pn, appends):
tinfoil.config_data)
+def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
+ """List contents of a git treeish"""
+ import bb
+ cmd = ['git', 'ls-tree', '-z', 'HEAD']
+ if recursive:
+ cmd.append('-r')
+ out, _ = bb.process.run(cmd, cwd=repodir)
+ return [line.split(None, 4)[3] for line in out.split('\0') if line]
+
+
def _ls_tree(directory):
"""Recursive listing of files in a directory"""
ret = []
@@ -571,6 +581,35 @@ def update_recipe(args, config, basepath, workspace):
logger.error('Invalid hash returned by git: %s' % stdout)
return 1
+ # Find out local files (SRC_URI files that exist in the "recipe space").
+ # Local files that reside in srctree are not included in patch generation.
+ # Instead they are directly copied over the original source files (in
+ # recipe space).
+ #
+ # NOTE: "Filtering out" of local files in this way is not entirely reliable
+ # - we don't catch files that are deleted, for example. A more reliable way
+ # to implement this would be to use "negative pathspecs" which were
+ # introduced in Git v1.9.0. Revisit this when/if the required Git version
+ # becomes greater than that.
+ local_files = oe.recipeutils.get_recipe_local_files(rd)
+ tempdir = tempfile.mkdtemp(prefix='devtool')
+ try:
+ # Copy local files from srctree HEAD to "recipe space"
+ # Local files might be "all over the place", need recursive ls-tree
+ git_files = set(_git_ls_tree(srctree, recursive=True))
+ copy_files = git_files.intersection(set(local_files.keys()))
+ patch_include_paths = git_files.difference(set(local_files.keys()))
+ bb.process.run(['git', 'checkout', 'HEAD', '--'] + list(copy_files),
+ cwd=srctree,
+ env=dict(os.environ, GIT_WORK_TREE=tempdir))
+ for fname in _ls_tree(tempdir):
+ logger.info('Updating file %s' % fname)
+ shutil.copy2(os.path.join(tempdir, fname),
+ local_files[fname])
+ finally:
+ shutil.rmtree(tempdir)
+
+ # Update recipe and patches
removepatches = []
destpath = None
if mode == 'srcrev':
@@ -585,7 +624,8 @@ def update_recipe(args, config, basepath, workspace):
old_srcrev = (rd.getVar('SRCREV', False) or '')
tempdir = tempfile.mkdtemp(prefix='devtool')
try:
- GitApplyTree.extractPatches(srctree, old_srcrev, tempdir)
+ GitApplyTree.extractPatches(srctree, old_srcrev, tempdir,
+ patch_include_paths)
newpatches = os.listdir(tempdir)
for patch in existing_patches:
patchfile = os.path.basename(patch)
@@ -645,7 +685,8 @@ def update_recipe(args, config, basepath, workspace):
# Get all patches from source tree and check if any should be removed
tempdir = tempfile.mkdtemp(prefix='devtool')
try:
- GitApplyTree.extractPatches(srctree, initial_rev, tempdir)
+ GitApplyTree.extractPatches(srctree, initial_rev, tempdir,
+ patch_include_paths)
newpatches = os.listdir(tempdir)
for patch in existing_patches:
# If it's a git sequence named patch, the numbers might not match up
@@ -667,7 +708,8 @@ def update_recipe(args, config, basepath, workspace):
# Get updated patches from source tree
tempdir = tempfile.mkdtemp(prefix='devtool')
try:
- GitApplyTree.extractPatches(srctree, update_rev, tempdir)
+ GitApplyTree.extractPatches(srctree, update_rev, tempdir,
+ patch_include_paths)
# Match up and replace existing patches with corresponding new patches
updatepatches = False
--
2.1.4
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 6/7] devtool: extract: always import local files to srctree
2015-06-03 14:50 [PATCH v2 0/7] devtool: improve handling of local source files Markus Lehtonen
` (4 preceding siblings ...)
2015-06-03 14:50 ` [PATCH v2 5/7] devtool: update-recipe: update local files directly Markus Lehtonen
@ 2015-06-03 14:50 ` Markus Lehtonen
2015-06-03 14:50 ` [PATCH v2 7/7] devtool: modify: make bitbake use local files from srctree Markus Lehtonen
2015-06-04 10:49 ` [PATCH v2 0/7] devtool: improve handling of local source files Paul Eggleton
7 siblings, 0 replies; 10+ messages in thread
From: Markus Lehtonen @ 2015-06-03 14:50 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
Import all non-compressed/non-arcived non-remote source files (i.e.
local files from the SRC_URI) - excluding patches - to the srctree
repository. The files will be placed in a subdirectory called
'local-files'. However, in case S=WORKDIR, the files are imported into
root ot srctree (and not under 'local-files'), just like before.
[YOCTO #7602]
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/selftest/devtool.py | 46 +++++++++++++++++++++++++++++++
scripts/lib/devtool/standard.py | 58 ++++++++++++++++++++++++++++++---------
2 files changed, 91 insertions(+), 13 deletions(-)
diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
index 6aaf5a5..c1623c2 100644
--- a/meta/lib/oeqa/selftest/devtool.py
+++ b/meta/lib/oeqa/selftest/devtool.py
@@ -716,6 +716,52 @@ class DevtoolTests(DevtoolBase):
raise AssertionError('Unexpected modified file in status: %s' %
line)
+ def test_devtool_update_recipe_local_files_2(self):
+ """Check handling of local source files for recipes where
+ S != WORKDIR"""
+ workspacedir = self._get_workspace_dir()
+ testrecipe = 'lzo'
+ recipefile = get_bb_var('FILE', testrecipe)
+ # Setup srctree for modifying the recipe
+ tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+ self.track_for_cleanup(tempdir)
+ self.track_for_cleanup(workspacedir)
+ self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+ result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
+ # Check git repo
+ self._check_src_repo(tempdir)
+ # Edit / commit local source
+ runCmd('echo "# New comment" >> local-files/acinclude.m4', cwd=tempdir)
+ runCmd('git commit -am "Edit existing file"', cwd=tempdir)
+ runCmd('echo "Foo" > local-files/new-local', cwd=tempdir)
+ runCmd('git add local-files/new-local', cwd=tempdir)
+ runCmd('git commit -m "Add new local file"', cwd=tempdir)
+ runCmd('echo "Foo" > new-file', cwd=tempdir)
+ runCmd('git add new-file', cwd=tempdir)
+ runCmd('git commit -m "Add new file"', cwd=tempdir)
+ self.add_command_to_tearDown('cd %s; git clean -fd .; git checkout .' %
+ os.path.dirname(recipefile))
+ runCmd('devtool update-recipe %s' % testrecipe)
+ result = runCmd('git status . --porcelain',
+ cwd=os.path.dirname(recipefile))
+ status = result.output.splitlines()
+ self.assertEqual(len(status), 3,
+ 'Less/more files modified than expected. '
+ 'Entire status:\n%s' % result.output)
+ for line in status:
+ if line.endswith('acinclude.m4'):
+ self.assertEqual(line[:3], ' M ',
+ 'Unexpected status in line: %s' % line)
+ elif line.endswith('Add-new-file.patch'):
+ self.assertEqual(line[:3], '?? ',
+ 'Unexpected status in line: %s' % line)
+ elif re.search('%s_[^_]*.bb$' % testrecipe, line):
+ self.assertEqual(line[:3], ' M ',
+ 'Unexpected status in line: %s' % line)
+ else:
+ raise AssertionError('Unexpected modified file in status: %s' %
+ line)
+
def test_devtool_extract(self):
workspacedir = self._get_workspace_dir()
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 75cc495..44948fd 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -303,10 +303,11 @@ def _extract_source(srctree, keep_temp, devbranch, d):
logger.info('Unpacking...')
exec_task_func('do_unpack', False)
srcsubdir = crd.getVar('S', True)
+
+ recipe_patches = [os.path.basename(patch) for patch in
+ oe.recipeutils.get_recipe_patches(crd)]
if srcsubdir == workdir:
# Find non-patch sources that were "unpacked" to srctree directory
- recipe_patches = [os.path.basename(patch) for patch in
- oe.recipeutils.get_recipe_patches(crd)]
src_files = [fname for fname in _ls_tree(workdir) if
os.path.basename(fname) not in recipe_patches]
# Force separate S so that patch files can be left out from srctree
@@ -357,6 +358,23 @@ def _extract_source(srctree, keep_temp, devbranch, d):
bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
+ # Add unpacked local files (in case of S=WORKDIR these were already
+ # moved to S) into srctree
+ local_files = oe.recipeutils.get_recipe_local_files(crd)
+ local_files = [fname for fname in local_files if
+ os.path.basename(fname) not in recipe_patches and
+ os.path.exists(os.path.join(workdir, fname))]
+ if local_files:
+ logger.info('Adding local files...')
+ for fname in local_files:
+ localf_subdir = os.path.join(srcsubdir, 'local-files',
+ os.path.dirname(fname))
+ bb.utils.mkdirhier(localf_subdir)
+ shutil.move(os.path.join(workdir, fname), localf_subdir)
+ bb.process.run(['git', 'add', 'local-files'], cwd=srcsubdir)
+ bb.process.run(['git', 'commit', '-q', '-m', 'Add local files'],
+ cwd=srcsubdir)
+
if os.path.exists(patchdir):
shutil.rmtree(patchdir)
if haspatches:
@@ -595,17 +613,31 @@ def update_recipe(args, config, basepath, workspace):
tempdir = tempfile.mkdtemp(prefix='devtool')
try:
# Copy local files from srctree HEAD to "recipe space"
- # Local files might be "all over the place", need recursive ls-tree
- git_files = set(_git_ls_tree(srctree, recursive=True))
- copy_files = git_files.intersection(set(local_files.keys()))
- patch_include_paths = git_files.difference(set(local_files.keys()))
- bb.process.run(['git', 'checkout', 'HEAD', '--'] + list(copy_files),
- cwd=srctree,
- env=dict(os.environ, GIT_WORK_TREE=tempdir))
- for fname in _ls_tree(tempdir):
- logger.info('Updating file %s' % fname)
- shutil.copy2(os.path.join(tempdir, fname),
- local_files[fname])
+ if os.path.isdir(os.path.join(srctree, 'local-files')):
+ # Local files are in 'local-files' only list root of git repo
+ git_files = set(_git_ls_tree(srctree))
+ patch_include_paths = git_files.difference(set(['local-files']))
+ bb.process.run(['git', 'checkout', 'HEAD', '--', 'local-files'],
+ cwd=srctree,
+ env=dict(os.environ, GIT_WORK_TREE=tempdir))
+ local_src_dir = os.path.join(tempdir, 'local-files')
+ else:
+ # Local files might be "all over the place", need recursive ls-tree
+ git_files = set(_git_ls_tree(srctree, recursive=True))
+ copy_files = git_files.intersection(set(local_files.keys()))
+ patch_include_paths = git_files.difference(set(local_files.keys()))
+ bb.process.run(['git', 'checkout', 'HEAD', '--'] + list(copy_files),
+ cwd=srctree,
+ env=dict(os.environ, GIT_WORK_TREE=tempdir))
+ local_src_dir = tempdir
+
+ for fname in _ls_tree(local_src_dir):
+ if fname in local_files:
+ logger.info('Updating file %s' % fname)
+ shutil.copy2(os.path.join(local_src_dir, fname),
+ local_files[fname])
+ else:
+ logger.warning('File %s not in SRC_URI, skipping it' % fname)
finally:
shutil.rmtree(tempdir)
--
2.1.4
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 7/7] devtool: modify: make bitbake use local files from srctree
2015-06-03 14:50 [PATCH v2 0/7] devtool: improve handling of local source files Markus Lehtonen
` (5 preceding siblings ...)
2015-06-03 14:50 ` [PATCH v2 6/7] devtool: extract: always import local files to srctree Markus Lehtonen
@ 2015-06-03 14:50 ` Markus Lehtonen
2015-06-04 10:49 ` [PATCH v2 0/7] devtool: improve handling of local source files Paul Eggleton
7 siblings, 0 replies; 10+ messages in thread
From: Markus Lehtonen @ 2015-06-03 14:50 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
This change makes it possible to have local files (non-remote SRC_URI
files, i.e. files that are located in the "recipe space") under the
srctree even if S!=WORKDIR. The files must be placed under the
'local-files' subdirectory.
Complements the previous patch that imports local files into srctree.
[YOCTO #7602]
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
scripts/lib/devtool/standard.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 44948fd..5d2a264 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -501,8 +501,13 @@ def modify(args, config, basepath, workspace):
appendname = re.sub(r'_.*', '_%', appendname)
appendfile = os.path.join(appendpath, appendname + '.bbappend')
with open(appendfile, 'w') as f:
- f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n')
- f.write('inherit externalsrc\n')
+ f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n')
+ # Local files can be modified/tracked in separate subdir under srctree
+ # Mostly useful for packages with S != WORKDIR
+ f.write('FILESPATH_prepend := "%s:"\n' %
+ os.path.join(srctree, 'local-files'))
+
+ f.write('\ninherit externalsrc\n')
f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n')
f.write('EXTERNALSRC_pn-%s = "%s"\n' % (args.recipename, srctree))
--
2.1.4
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [PATCH v2 0/7] devtool: improve handling of local source files
2015-06-03 14:50 [PATCH v2 0/7] devtool: improve handling of local source files Markus Lehtonen
` (6 preceding siblings ...)
2015-06-03 14:50 ` [PATCH v2 7/7] devtool: modify: make bitbake use local files from srctree Markus Lehtonen
@ 2015-06-04 10:49 ` Paul Eggleton
2015-06-04 13:14 ` Markus Lehtonen
7 siblings, 1 reply; 10+ messages in thread
From: Paul Eggleton @ 2015-06-04 10:49 UTC (permalink / raw)
To: Markus Lehtonen; +Cc: openembedded-core
Hi Markus,
On Wednesday 03 June 2015 17:50:25 Markus Lehtonen wrote:
> Second version of my patchset aiming to improve handling of local source
> files. The only significant change since the first version is the addition
> of unit tests.
Did you see the comments I made on v1 patchset?
https://www.mail-archive.com/openembedded-core@lists.openembedded.org/msg64396.html
Cheers,
Paul
--
Paul Eggleton
Intel Open Source Technology Centre
^ permalink raw reply [flat|nested] 10+ messages in thread