* [PATCH v3 01/10] recipeutils: implement get_recipe_local_files()
2015-09-24 11:52 [PATCH v3 00/10] devtool: improve handling of local source files Markus Lehtonen
@ 2015-09-24 11:52 ` Markus Lehtonen
2015-09-24 11:52 ` [PATCH v3 02/10] oe.patch.GitApplyTree: add paths argument to extractPatches Markus Lehtonen
` (8 subsequent siblings)
9 siblings, 0 replies; 18+ messages in thread
From: Markus Lehtonen @ 2015-09-24 11:52 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oe/recipeutils.py | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index 35b88d3..56056db 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -336,6 +336,22 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True):
return remotes
+def get_recipe_local_files(d, patches=False):
+ """Get a list of local files in SRC_URI within a recipe."""
+ uris = (d.getVar('SRC_URI', True) or "").split()
+ fetch = bb.fetch2.Fetch(uris, d)
+ ret = {}
+ for uri in uris:
+ if fetch.ud[uri].type == 'file':
+ if (not patches and
+ bb.utils.exec_flat_python_func('patch_path', uri, fetch, '')):
+ continue
+ # Skip files that are referenced by absolute path
+ if not os.path.isabs(fetch.ud[uri].basepath):
+ ret[fetch.ud[uri].basepath] = fetch.localpath(uri)
+ return ret
+
+
def get_recipe_patches(d):
"""Get a list of the patches included in SRC_URI within a recipe."""
patchfiles = []
--
2.1.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH v3 02/10] oe.patch.GitApplyTree: add paths argument to extractPatches
2015-09-24 11:52 [PATCH v3 00/10] devtool: improve handling of local source files Markus Lehtonen
2015-09-24 11:52 ` [PATCH v3 01/10] recipeutils: implement get_recipe_local_files() Markus Lehtonen
@ 2015-09-24 11:52 ` Markus Lehtonen
2015-09-24 11:53 ` [PATCH v3 03/10] oe-selftest: devtool: add method for checking workspace dir Markus Lehtonen
` (7 subsequent siblings)
9 siblings, 0 replies; 18+ messages in thread
From: Markus Lehtonen @ 2015-09-24 11:52 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
Makes it possible to define which paths are included in the patches.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oe/patch.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/meta/lib/oe/patch.py b/meta/lib/oe/patch.py
index 7441214..2bf501e 100644
--- a/meta/lib/oe/patch.py
+++ b/meta/lib/oe/patch.py
@@ -337,12 +337,15 @@ class GitApplyTree(PatchTree):
return (tmpfile, cmd)
@staticmethod
- def extractPatches(tree, startcommit, outdir):
+ def extractPatches(tree, startcommit, outdir, paths=None):
import tempfile
import shutil
tempdir = tempfile.mkdtemp(prefix='oepatch')
try:
shellcmd = ["git", "format-patch", startcommit, "-o", tempdir]
+ if paths:
+ shellcmd.append('--')
+ shellcmd.extend(paths)
out = runcmd(["sh", "-c", " ".join(shellcmd)], tree)
if out:
for srcfile in out.split():
--
2.1.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH v3 03/10] oe-selftest: devtool: add method for checking workspace dir
2015-09-24 11:52 [PATCH v3 00/10] devtool: improve handling of local source files Markus Lehtonen
2015-09-24 11:52 ` [PATCH v3 01/10] recipeutils: implement get_recipe_local_files() Markus Lehtonen
2015-09-24 11:52 ` [PATCH v3 02/10] oe.patch.GitApplyTree: add paths argument to extractPatches Markus Lehtonen
@ 2015-09-24 11:53 ` Markus Lehtonen
2015-09-28 20:25 ` Leonardo Sandoval
2015-09-24 11:53 ` [PATCH v3 04/10] oe-selftest: devtool: add method for checking srctree repo Markus Lehtonen
` (6 subsequent siblings)
9 siblings, 1 reply; 18+ messages in thread
From: Markus Lehtonen @ 2015-09-24 11:53 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
In order to remove some code duplication.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/selftest/devtool.py | 63 +++++++++++++++------------------------
1 file changed, 24 insertions(+), 39 deletions(-)
diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
index 3a8168c..b8b872c 100644
--- a/meta/lib/oeqa/selftest/devtool.py
+++ b/meta/lib/oeqa/selftest/devtool.py
@@ -84,11 +84,18 @@ class DevtoolBase(oeSelfTest):
class DevtoolTests(DevtoolBase):
+ def _get_workspace_dir(self):
+ """Get workspace directory"""
+ workspacedir = os.path.join(self.builddir, 'workspace')
+ self.assertTrue(not os.path.exists(workspacedir),
+ 'This test cannot be run with a workspace directory '
+ 'under the build directory')
+ return workspacedir
+
@testcase(1158)
def test_create_workspace(self):
# Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
result = runCmd('bitbake-layers show-layers')
self.assertTrue('/workspace' not in result.output, 'This test cannot be run with a workspace layer in bblayers.conf')
# Try creating a workspace layer with a specific path
@@ -109,9 +116,7 @@ class DevtoolTests(DevtoolBase):
@testcase(1159)
def test_devtool_add(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
# Fetch source
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
self.track_for_cleanup(tempdir)
@@ -144,9 +149,7 @@ class DevtoolTests(DevtoolBase):
@testcase(1162)
def test_devtool_add_library(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
# We don't have the ability to pick up this dependency automatically yet...
bitbake('libusb1')
# Fetch source
@@ -185,9 +188,7 @@ class DevtoolTests(DevtoolBase):
@testcase(1160)
def test_devtool_add_fetch(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
# Fetch source
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
self.track_for_cleanup(tempdir)
@@ -232,9 +233,7 @@ class DevtoolTests(DevtoolBase):
@testcase(1161)
def test_devtool_add_fetch_git(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
# Fetch source
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
self.track_for_cleanup(tempdir)
@@ -284,9 +283,7 @@ class DevtoolTests(DevtoolBase):
@testcase(1164)
def test_devtool_modify(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
# Clean up anything in the workdir/sysroot/sstate cache
bitbake('mdadm -c cleansstate')
# Try modifying a recipe
@@ -336,9 +333,7 @@ class DevtoolTests(DevtoolBase):
@testcase(1166)
def test_devtool_modify_invalid(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
# Try modifying some recipes
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
self.track_for_cleanup(tempdir)
@@ -400,8 +395,7 @@ class DevtoolTests(DevtoolBase):
@testcase(1165)
def test_devtool_modify_git(self):
# Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
testrecipe = 'mkelfimage'
src_uri = get_bb_var('SRC_URI', testrecipe)
self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)
@@ -434,8 +428,7 @@ class DevtoolTests(DevtoolBase):
@testcase(1167)
def test_devtool_modify_localfiles(self):
# Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
testrecipe = 'lighttpd'
src_uri = (get_bb_var('SRC_URI', testrecipe) or '').split()
foundlocal = False
@@ -467,8 +460,7 @@ class DevtoolTests(DevtoolBase):
@testcase(1169)
def test_devtool_update_recipe(self):
# Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
testrecipe = 'minicom'
recipefile = get_bb_var('FILE', testrecipe)
src_uri = get_bb_var('SRC_URI', testrecipe)
@@ -514,8 +506,7 @@ class DevtoolTests(DevtoolBase):
@testcase(1172)
def test_devtool_update_recipe_git(self):
# Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
testrecipe = 'mtd-utils'
recipefile = get_bb_var('FILE', testrecipe)
src_uri = get_bb_var('SRC_URI', testrecipe)
@@ -609,8 +600,7 @@ class DevtoolTests(DevtoolBase):
@testcase(1170)
def test_devtool_update_recipe_append(self):
# Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
testrecipe = 'mdadm'
recipefile = get_bb_var('FILE', testrecipe)
src_uri = get_bb_var('SRC_URI', testrecipe)
@@ -685,8 +675,7 @@ class DevtoolTests(DevtoolBase):
@testcase(1171)
def test_devtool_update_recipe_append_git(self):
# Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
testrecipe = 'mtd-utils'
recipefile = get_bb_var('FILE', testrecipe)
src_uri = get_bb_var('SRC_URI', testrecipe)
@@ -781,9 +770,7 @@ class DevtoolTests(DevtoolBase):
@testcase(1163)
def test_devtool_extract(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
# Try devtool extract
self.track_for_cleanup(tempdir)
@@ -795,9 +782,7 @@ class DevtoolTests(DevtoolBase):
@testcase(1168)
def test_devtool_reset_all(self):
- # Check preconditions
- workspacedir = os.path.join(self.builddir, 'workspace')
- self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ workspacedir = self._get_workspace_dir()
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
self.track_for_cleanup(tempdir)
self.track_for_cleanup(workspacedir)
@@ -846,7 +831,7 @@ class DevtoolTests(DevtoolBase):
break
else:
self.skipTest('No tap devices found - you must set up tap devices with scripts/runqemu-gen-tapdevs before running this test')
- workspacedir = os.path.join(self.builddir, 'workspace')
+ workspacedir = self._get_workspace_dir()
self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
# Definitions
testrecipe = 'mdadm'
--
2.1.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* Re: [PATCH v3 03/10] oe-selftest: devtool: add method for checking workspace dir
2015-09-24 11:53 ` [PATCH v3 03/10] oe-selftest: devtool: add method for checking workspace dir Markus Lehtonen
@ 2015-09-28 20:25 ` Leonardo Sandoval
2015-09-29 10:57 ` Markus Lehtonen
0 siblings, 1 reply; 18+ messages in thread
From: Leonardo Sandoval @ 2015-09-28 20:25 UTC (permalink / raw)
To: Markus Lehtonen, openembedded-core; +Cc: Paul Eggleton
On 09/24/2015 06:53 AM, Markus Lehtonen wrote:
> In order to remove some code duplication.
>
> Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
> ---
> meta/lib/oeqa/selftest/devtool.py | 63 +++++++++++++++------------------------
> 1 file changed, 24 insertions(+), 39 deletions(-)
>
> diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
> index 3a8168c..b8b872c 100644
> --- a/meta/lib/oeqa/selftest/devtool.py
> +++ b/meta/lib/oeqa/selftest/devtool.py
> @@ -84,11 +84,18 @@ class DevtoolBase(oeSelfTest):
>
> class DevtoolTests(DevtoolBase):
>
> + def _get_workspace_dir(self):
> + """Get workspace directory"""
> + workspacedir = os.path.join(self.builddir, 'workspace')
> + self.assertTrue(not os.path.exists(workspacedir),
> + 'This test cannot be run with a workspace directory '
> + 'under the build directory')
> + return workspacedir
> +
> @testcase(1158)
> def test_create_workspace(self):
> # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
If all tests are using workspacedir, I believe it make sense to have a
setUp method and setting workspacedir there:
.
def setUp(self):
self.workspacedir = # the _get_workspace_dir body code goes here
.
.
> result = runCmd('bitbake-layers show-layers')
> self.assertTrue('/workspace' not in result.output, 'This test cannot be run with a workspace layer in bblayers.conf')
> # Try creating a workspace layer with a specific path
> @@ -109,9 +116,7 @@ class DevtoolTests(DevtoolBase):
>
> @testcase(1159)
> def test_devtool_add(self):
> - # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
> # Fetch source
> tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> self.track_for_cleanup(tempdir)
> @@ -144,9 +149,7 @@ class DevtoolTests(DevtoolBase):
>
> @testcase(1162)
> def test_devtool_add_library(self):
> - # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
> # We don't have the ability to pick up this dependency automatically yet...
> bitbake('libusb1')
> # Fetch source
> @@ -185,9 +188,7 @@ class DevtoolTests(DevtoolBase):
>
> @testcase(1160)
> def test_devtool_add_fetch(self):
> - # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
> # Fetch source
> tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> self.track_for_cleanup(tempdir)
> @@ -232,9 +233,7 @@ class DevtoolTests(DevtoolBase):
>
> @testcase(1161)
> def test_devtool_add_fetch_git(self):
> - # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
> # Fetch source
> tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> self.track_for_cleanup(tempdir)
> @@ -284,9 +283,7 @@ class DevtoolTests(DevtoolBase):
>
> @testcase(1164)
> def test_devtool_modify(self):
> - # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
> # Clean up anything in the workdir/sysroot/sstate cache
> bitbake('mdadm -c cleansstate')
> # Try modifying a recipe
> @@ -336,9 +333,7 @@ class DevtoolTests(DevtoolBase):
>
> @testcase(1166)
> def test_devtool_modify_invalid(self):
> - # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
> # Try modifying some recipes
> tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> self.track_for_cleanup(tempdir)
> @@ -400,8 +395,7 @@ class DevtoolTests(DevtoolBase):
> @testcase(1165)
> def test_devtool_modify_git(self):
> # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
> testrecipe = 'mkelfimage'
> src_uri = get_bb_var('SRC_URI', testrecipe)
> self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)
> @@ -434,8 +428,7 @@ class DevtoolTests(DevtoolBase):
> @testcase(1167)
> def test_devtool_modify_localfiles(self):
> # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
> testrecipe = 'lighttpd'
> src_uri = (get_bb_var('SRC_URI', testrecipe) or '').split()
> foundlocal = False
> @@ -467,8 +460,7 @@ class DevtoolTests(DevtoolBase):
> @testcase(1169)
> def test_devtool_update_recipe(self):
> # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
> testrecipe = 'minicom'
> recipefile = get_bb_var('FILE', testrecipe)
> src_uri = get_bb_var('SRC_URI', testrecipe)
> @@ -514,8 +506,7 @@ class DevtoolTests(DevtoolBase):
> @testcase(1172)
> def test_devtool_update_recipe_git(self):
> # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
> testrecipe = 'mtd-utils'
> recipefile = get_bb_var('FILE', testrecipe)
> src_uri = get_bb_var('SRC_URI', testrecipe)
> @@ -609,8 +600,7 @@ class DevtoolTests(DevtoolBase):
> @testcase(1170)
> def test_devtool_update_recipe_append(self):
> # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
> testrecipe = 'mdadm'
> recipefile = get_bb_var('FILE', testrecipe)
> src_uri = get_bb_var('SRC_URI', testrecipe)
> @@ -685,8 +675,7 @@ class DevtoolTests(DevtoolBase):
> @testcase(1171)
> def test_devtool_update_recipe_append_git(self):
> # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
> testrecipe = 'mtd-utils'
> recipefile = get_bb_var('FILE', testrecipe)
> src_uri = get_bb_var('SRC_URI', testrecipe)
> @@ -781,9 +770,7 @@ class DevtoolTests(DevtoolBase):
>
> @testcase(1163)
> def test_devtool_extract(self):
> - # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
> tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> # Try devtool extract
> self.track_for_cleanup(tempdir)
> @@ -795,9 +782,7 @@ class DevtoolTests(DevtoolBase):
>
> @testcase(1168)
> def test_devtool_reset_all(self):
> - # Check preconditions
> - workspacedir = os.path.join(self.builddir, 'workspace')
> - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> + workspacedir = self._get_workspace_dir()
> tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> self.track_for_cleanup(tempdir)
> self.track_for_cleanup(workspacedir)
> @@ -846,7 +831,7 @@ class DevtoolTests(DevtoolBase):
> break
> else:
> self.skipTest('No tap devices found - you must set up tap devices with scripts/runqemu-gen-tapdevs before running this test')
> - workspacedir = os.path.join(self.builddir, 'workspace')
> + workspacedir = self._get_workspace_dir()
> self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> # Definitions
> testrecipe = 'mdadm'
>
^ permalink raw reply [flat|nested] 18+ messages in thread* Re: [PATCH v3 03/10] oe-selftest: devtool: add method for checking workspace dir
2015-09-28 20:25 ` Leonardo Sandoval
@ 2015-09-29 10:57 ` Markus Lehtonen
2015-09-29 12:38 ` Markus Lehtonen
0 siblings, 1 reply; 18+ messages in thread
From: Markus Lehtonen @ 2015-09-29 10:57 UTC (permalink / raw)
To: Leonardo Sandoval; +Cc: Paul Eggleton, openembedded-core
Hi,
On Mon, 2015-09-28 at 15:25 -0500, Leonardo Sandoval wrote:
>
> On 09/24/2015 06:53 AM, Markus Lehtonen wrote:
> > In order to remove some code duplication.
> >
> > Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
> > ---
> > meta/lib/oeqa/selftest/devtool.py | 63 +++++++++++++++------------------------
> > 1 file changed, 24 insertions(+), 39 deletions(-)
> >
> > diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
> > index 3a8168c..b8b872c 100644
> > --- a/meta/lib/oeqa/selftest/devtool.py
> > +++ b/meta/lib/oeqa/selftest/devtool.py
> > @@ -84,11 +84,18 @@ class DevtoolBase(oeSelfTest):
> >
> > class DevtoolTests(DevtoolBase):
> >
> > + def _get_workspace_dir(self):
> > + """Get workspace directory"""
> > + workspacedir = os.path.join(self.builddir, 'workspace')
> > + self.assertTrue(not os.path.exists(workspacedir),
> > + 'This test cannot be run with a workspace directory '
> > + 'under the build directory')
> > + return workspacedir
> > +
> > @testcase(1158)
> > def test_create_workspace(self):
> > # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
>
> If all tests are using workspacedir, I believe it make sense to have a
> setUp method and setting workspacedir there:
>
> .
> def setUp(self):
> self.workspacedir = # the _get_workspace_dir body code goes here
> .
> .
Good point! Yes, I think this check is in every single test case so a
setup() method is nicer.
Thanks,
Markus
> > result = runCmd('bitbake-layers show-layers')
> > self.assertTrue('/workspace' not in result.output, 'This test cannot be run with a workspace layer in bblayers.conf')
> > # Try creating a workspace layer with a specific path
> > @@ -109,9 +116,7 @@ class DevtoolTests(DevtoolBase):
> >
> > @testcase(1159)
> > def test_devtool_add(self):
> > - # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
> > # Fetch source
> > tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> > self.track_for_cleanup(tempdir)
> > @@ -144,9 +149,7 @@ class DevtoolTests(DevtoolBase):
> >
> > @testcase(1162)
> > def test_devtool_add_library(self):
> > - # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
> > # We don't have the ability to pick up this dependency automatically yet...
> > bitbake('libusb1')
> > # Fetch source
> > @@ -185,9 +188,7 @@ class DevtoolTests(DevtoolBase):
> >
> > @testcase(1160)
> > def test_devtool_add_fetch(self):
> > - # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
> > # Fetch source
> > tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> > self.track_for_cleanup(tempdir)
> > @@ -232,9 +233,7 @@ class DevtoolTests(DevtoolBase):
> >
> > @testcase(1161)
> > def test_devtool_add_fetch_git(self):
> > - # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
> > # Fetch source
> > tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> > self.track_for_cleanup(tempdir)
> > @@ -284,9 +283,7 @@ class DevtoolTests(DevtoolBase):
> >
> > @testcase(1164)
> > def test_devtool_modify(self):
> > - # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
> > # Clean up anything in the workdir/sysroot/sstate cache
> > bitbake('mdadm -c cleansstate')
> > # Try modifying a recipe
> > @@ -336,9 +333,7 @@ class DevtoolTests(DevtoolBase):
> >
> > @testcase(1166)
> > def test_devtool_modify_invalid(self):
> > - # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
> > # Try modifying some recipes
> > tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> > self.track_for_cleanup(tempdir)
> > @@ -400,8 +395,7 @@ class DevtoolTests(DevtoolBase):
> > @testcase(1165)
> > def test_devtool_modify_git(self):
> > # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
> > testrecipe = 'mkelfimage'
> > src_uri = get_bb_var('SRC_URI', testrecipe)
> > self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)
> > @@ -434,8 +428,7 @@ class DevtoolTests(DevtoolBase):
> > @testcase(1167)
> > def test_devtool_modify_localfiles(self):
> > # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
> > testrecipe = 'lighttpd'
> > src_uri = (get_bb_var('SRC_URI', testrecipe) or '').split()
> > foundlocal = False
> > @@ -467,8 +460,7 @@ class DevtoolTests(DevtoolBase):
> > @testcase(1169)
> > def test_devtool_update_recipe(self):
> > # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
> > testrecipe = 'minicom'
> > recipefile = get_bb_var('FILE', testrecipe)
> > src_uri = get_bb_var('SRC_URI', testrecipe)
> > @@ -514,8 +506,7 @@ class DevtoolTests(DevtoolBase):
> > @testcase(1172)
> > def test_devtool_update_recipe_git(self):
> > # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
> > testrecipe = 'mtd-utils'
> > recipefile = get_bb_var('FILE', testrecipe)
> > src_uri = get_bb_var('SRC_URI', testrecipe)
> > @@ -609,8 +600,7 @@ class DevtoolTests(DevtoolBase):
> > @testcase(1170)
> > def test_devtool_update_recipe_append(self):
> > # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
> > testrecipe = 'mdadm'
> > recipefile = get_bb_var('FILE', testrecipe)
> > src_uri = get_bb_var('SRC_URI', testrecipe)
> > @@ -685,8 +675,7 @@ class DevtoolTests(DevtoolBase):
> > @testcase(1171)
> > def test_devtool_update_recipe_append_git(self):
> > # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
> > testrecipe = 'mtd-utils'
> > recipefile = get_bb_var('FILE', testrecipe)
> > src_uri = get_bb_var('SRC_URI', testrecipe)
> > @@ -781,9 +770,7 @@ class DevtoolTests(DevtoolBase):
> >
> > @testcase(1163)
> > def test_devtool_extract(self):
> > - # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
> > tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> > # Try devtool extract
> > self.track_for_cleanup(tempdir)
> > @@ -795,9 +782,7 @@ class DevtoolTests(DevtoolBase):
> >
> > @testcase(1168)
> > def test_devtool_reset_all(self):
> > - # Check preconditions
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > + workspacedir = self._get_workspace_dir()
> > tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> > self.track_for_cleanup(tempdir)
> > self.track_for_cleanup(workspacedir)
> > @@ -846,7 +831,7 @@ class DevtoolTests(DevtoolBase):
> > break
> > else:
> > self.skipTest('No tap devices found - you must set up tap devices with scripts/runqemu-gen-tapdevs before running this test')
> > - workspacedir = os.path.join(self.builddir, 'workspace')
> > + workspacedir = self._get_workspace_dir()
> > self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > # Definitions
> > testrecipe = 'mdadm'
> >
^ permalink raw reply [flat|nested] 18+ messages in thread* Re: [PATCH v3 03/10] oe-selftest: devtool: add method for checking workspace dir
2015-09-29 10:57 ` Markus Lehtonen
@ 2015-09-29 12:38 ` Markus Lehtonen
0 siblings, 0 replies; 18+ messages in thread
From: Markus Lehtonen @ 2015-09-29 12:38 UTC (permalink / raw)
To: Leonardo Sandoval; +Cc: Paul Eggleton, openembedded-core
On Tue, 2015-09-29 at 13:57 +0300, Markus Lehtonen wrote:
> Hi,
>
>
> On Mon, 2015-09-28 at 15:25 -0500, Leonardo Sandoval wrote:
> >
> > On 09/24/2015 06:53 AM, Markus Lehtonen wrote:
> > > In order to remove some code duplication.
> > >
> > > Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
> > > ---
> > > meta/lib/oeqa/selftest/devtool.py | 63 +++++++++++++++------------------------
> > > 1 file changed, 24 insertions(+), 39 deletions(-)
> > >
> > > diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
> > > index 3a8168c..b8b872c 100644
> > > --- a/meta/lib/oeqa/selftest/devtool.py
> > > +++ b/meta/lib/oeqa/selftest/devtool.py
> > > @@ -84,11 +84,18 @@ class DevtoolBase(oeSelfTest):
> > >
> > > class DevtoolTests(DevtoolBase):
> > >
> > > + def _get_workspace_dir(self):
> > > + """Get workspace directory"""
> > > + workspacedir = os.path.join(self.builddir, 'workspace')
> > > + self.assertTrue(not os.path.exists(workspacedir),
> > > + 'This test cannot be run with a workspace directory '
> > > + 'under the build directory')
> > > + return workspacedir
> > > +
> > > @testcase(1158)
> > > def test_create_workspace(self):
> > > # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> >
> > If all tests are using workspacedir, I believe it make sense to have a
> > setUp method and setting workspacedir there:
> >
> > .
> > def setUp(self):
> > self.workspacedir = # the _get_workspace_dir body code goes here
> > .
> > .
>
> Good point! Yes, I think this check is in every single test case so a
> setup() method is nicer.
An updated patchset is available at:
git://git.openembedded.org/openembedded-core-contrib
marquiz/devtool/localfiles
http://cgit.openembedded.org/openembedded-core-contrib/log/?h=marquiz/devtool/localfiles
Thanks,
Markus
>
> > > result = runCmd('bitbake-layers show-layers')
> > > self.assertTrue('/workspace' not in result.output, 'This test cannot be run with a workspace layer in bblayers.conf')
> > > # Try creating a workspace layer with a specific path
> > > @@ -109,9 +116,7 @@ class DevtoolTests(DevtoolBase):
> > >
> > > @testcase(1159)
> > > def test_devtool_add(self):
> > > - # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> > > # Fetch source
> > > tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> > > self.track_for_cleanup(tempdir)
> > > @@ -144,9 +149,7 @@ class DevtoolTests(DevtoolBase):
> > >
> > > @testcase(1162)
> > > def test_devtool_add_library(self):
> > > - # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> > > # We don't have the ability to pick up this dependency automatically yet...
> > > bitbake('libusb1')
> > > # Fetch source
> > > @@ -185,9 +188,7 @@ class DevtoolTests(DevtoolBase):
> > >
> > > @testcase(1160)
> > > def test_devtool_add_fetch(self):
> > > - # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> > > # Fetch source
> > > tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> > > self.track_for_cleanup(tempdir)
> > > @@ -232,9 +233,7 @@ class DevtoolTests(DevtoolBase):
> > >
> > > @testcase(1161)
> > > def test_devtool_add_fetch_git(self):
> > > - # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> > > # Fetch source
> > > tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> > > self.track_for_cleanup(tempdir)
> > > @@ -284,9 +283,7 @@ class DevtoolTests(DevtoolBase):
> > >
> > > @testcase(1164)
> > > def test_devtool_modify(self):
> > > - # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> > > # Clean up anything in the workdir/sysroot/sstate cache
> > > bitbake('mdadm -c cleansstate')
> > > # Try modifying a recipe
> > > @@ -336,9 +333,7 @@ class DevtoolTests(DevtoolBase):
> > >
> > > @testcase(1166)
> > > def test_devtool_modify_invalid(self):
> > > - # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> > > # Try modifying some recipes
> > > tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> > > self.track_for_cleanup(tempdir)
> > > @@ -400,8 +395,7 @@ class DevtoolTests(DevtoolBase):
> > > @testcase(1165)
> > > def test_devtool_modify_git(self):
> > > # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> > > testrecipe = 'mkelfimage'
> > > src_uri = get_bb_var('SRC_URI', testrecipe)
> > > self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)
> > > @@ -434,8 +428,7 @@ class DevtoolTests(DevtoolBase):
> > > @testcase(1167)
> > > def test_devtool_modify_localfiles(self):
> > > # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> > > testrecipe = 'lighttpd'
> > > src_uri = (get_bb_var('SRC_URI', testrecipe) or '').split()
> > > foundlocal = False
> > > @@ -467,8 +460,7 @@ class DevtoolTests(DevtoolBase):
> > > @testcase(1169)
> > > def test_devtool_update_recipe(self):
> > > # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> > > testrecipe = 'minicom'
> > > recipefile = get_bb_var('FILE', testrecipe)
> > > src_uri = get_bb_var('SRC_URI', testrecipe)
> > > @@ -514,8 +506,7 @@ class DevtoolTests(DevtoolBase):
> > > @testcase(1172)
> > > def test_devtool_update_recipe_git(self):
> > > # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> > > testrecipe = 'mtd-utils'
> > > recipefile = get_bb_var('FILE', testrecipe)
> > > src_uri = get_bb_var('SRC_URI', testrecipe)
> > > @@ -609,8 +600,7 @@ class DevtoolTests(DevtoolBase):
> > > @testcase(1170)
> > > def test_devtool_update_recipe_append(self):
> > > # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> > > testrecipe = 'mdadm'
> > > recipefile = get_bb_var('FILE', testrecipe)
> > > src_uri = get_bb_var('SRC_URI', testrecipe)
> > > @@ -685,8 +675,7 @@ class DevtoolTests(DevtoolBase):
> > > @testcase(1171)
> > > def test_devtool_update_recipe_append_git(self):
> > > # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> > > testrecipe = 'mtd-utils'
> > > recipefile = get_bb_var('FILE', testrecipe)
> > > src_uri = get_bb_var('SRC_URI', testrecipe)
> > > @@ -781,9 +770,7 @@ class DevtoolTests(DevtoolBase):
> > >
> > > @testcase(1163)
> > > def test_devtool_extract(self):
> > > - # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> > > tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> > > # Try devtool extract
> > > self.track_for_cleanup(tempdir)
> > > @@ -795,9 +782,7 @@ class DevtoolTests(DevtoolBase):
> > >
> > > @testcase(1168)
> > > def test_devtool_reset_all(self):
> > > - # Check preconditions
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > + workspacedir = self._get_workspace_dir()
> > > tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> > > self.track_for_cleanup(tempdir)
> > > self.track_for_cleanup(workspacedir)
> > > @@ -846,7 +831,7 @@ class DevtoolTests(DevtoolBase):
> > > break
> > > else:
> > > self.skipTest('No tap devices found - you must set up tap devices with scripts/runqemu-gen-tapdevs before running this test')
> > > - workspacedir = os.path.join(self.builddir, 'workspace')
> > > + workspacedir = self._get_workspace_dir()
> > > self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> > > # Definitions
> > > testrecipe = 'mdadm'
> > >
>
>
>
^ permalink raw reply [flat|nested] 18+ messages in thread
* [PATCH v3 04/10] oe-selftest: devtool: add method for checking srctree repo
2015-09-24 11:52 [PATCH v3 00/10] devtool: improve handling of local source files Markus Lehtonen
` (2 preceding siblings ...)
2015-09-24 11:53 ` [PATCH v3 03/10] oe-selftest: devtool: add method for checking workspace dir Markus Lehtonen
@ 2015-09-24 11:53 ` Markus Lehtonen
2015-09-24 11:53 ` [PATCH v3 05/10] oe-selftest: devtool: add method for checking repo status Markus Lehtonen
` (5 subsequent siblings)
9 siblings, 0 replies; 18+ messages in thread
From: Markus Lehtonen @ 2015-09-24 11:53 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
Removes some code duplication.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/selftest/devtool.py | 49 ++++++++++++++-------------------------
1 file changed, 18 insertions(+), 31 deletions(-)
diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
index b8b872c..f459a6d 100644
--- a/meta/lib/oeqa/selftest/devtool.py
+++ b/meta/lib/oeqa/selftest/devtool.py
@@ -92,6 +92,17 @@ class DevtoolTests(DevtoolBase):
'under the build directory')
return workspacedir
+ def _check_src_repo(self, repo_dir):
+ """Check srctree git repository"""
+ self.assertTrue(os.path.isdir(os.path.join(repo_dir, '.git')),
+ 'git repository for external source tree not found')
+ result = runCmd('git status --porcelain', cwd=repo_dir)
+ self.assertEqual(result.output.strip(), "",
+ 'Created git repo is not clean')
+ result = runCmd('git symbolic-ref HEAD', cwd=repo_dir)
+ self.assertEqual(result.output.strip(), "refs/heads/devtool",
+ 'Wrong branch in git repo')
+
@testcase(1158)
def test_create_workspace(self):
# Check preconditions
@@ -294,7 +305,6 @@ class DevtoolTests(DevtoolBase):
self.add_command_to_tearDown('bitbake -c clean mdadm')
result = runCmd('devtool modify mdadm -x %s' % tempdir)
self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found')
- self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')
self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')
matches = glob.glob(os.path.join(workspacedir, 'appends', 'mdadm_*.bbappend'))
self.assertTrue(matches, 'bbappend not created %s' % result.output)
@@ -303,10 +313,7 @@ class DevtoolTests(DevtoolBase):
self.assertIn('mdadm', result.output)
self.assertIn(tempdir, result.output)
# Check git repo
- result = runCmd('git status --porcelain', cwd=tempdir)
- self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')
- result = runCmd('git symbolic-ref HEAD', cwd=tempdir)
- self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')
+ self._check_src_repo(tempdir)
# Try building
bitbake('mdadm')
# Try making (minor) modifications to the source
@@ -409,7 +416,6 @@ class DevtoolTests(DevtoolBase):
self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)
result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found')
- self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')
self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. devtool output: %s' % result.output)
matches = glob.glob(os.path.join(workspacedir, 'appends', 'mkelfimage_*.bbappend'))
self.assertTrue(matches, 'bbappend not created')
@@ -418,10 +424,7 @@ class DevtoolTests(DevtoolBase):
self.assertIn(testrecipe, result.output)
self.assertIn(tempdir, result.output)
# Check git repo
- result = runCmd('git status --porcelain', cwd=tempdir)
- self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')
- result = runCmd('git symbolic-ref HEAD', cwd=tempdir)
- self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')
+ self._check_src_repo(tempdir)
# Try building
bitbake(testrecipe)
@@ -475,11 +478,7 @@ class DevtoolTests(DevtoolBase):
# (don't bother with cleaning the recipe on teardown, we won't be building it)
result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
# Check git repo
- self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')
- result = runCmd('git status --porcelain', cwd=tempdir)
- self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')
- result = runCmd('git symbolic-ref HEAD', cwd=tempdir)
- self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')
+ self._check_src_repo(tempdir)
# Add a couple of commits
# FIXME: this only tests adding, need to also test update and remove
result = runCmd('echo "Additional line" >> README', cwd=tempdir)
@@ -526,11 +525,7 @@ class DevtoolTests(DevtoolBase):
# (don't bother with cleaning the recipe on teardown, we won't be building it)
result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
# Check git repo
- self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')
- result = runCmd('git status --porcelain', cwd=tempdir)
- self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')
- result = runCmd('git symbolic-ref HEAD', cwd=tempdir)
- self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')
+ self._check_src_repo(tempdir)
# Add a couple of commits
# FIXME: this only tests adding, need to also test update and remove
result = runCmd('echo "# Additional line" >> Makefile', cwd=tempdir)
@@ -617,11 +612,7 @@ class DevtoolTests(DevtoolBase):
# (don't bother with cleaning the recipe on teardown, we won't be building it)
result = runCmd('devtool modify %s -x %s' % (testrecipe, tempsrcdir))
# Check git repo
- self.assertTrue(os.path.isdir(os.path.join(tempsrcdir, '.git')), 'git repository for external source tree not found')
- result = runCmd('git status --porcelain', cwd=tempsrcdir)
- self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')
- result = runCmd('git symbolic-ref HEAD', cwd=tempsrcdir)
- self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')
+ self._check_src_repo(tempsrcdir)
# Add a commit
result = runCmd("sed 's!\\(#define VERSION\\W*\"[^\"]*\\)\"!\\1-custom\"!' -i ReadMe.c", cwd=tempsrcdir)
result = runCmd('git commit -a -m "Add our custom version"', cwd=tempsrcdir)
@@ -696,11 +687,7 @@ class DevtoolTests(DevtoolBase):
# (don't bother with cleaning the recipe on teardown, we won't be building it)
result = runCmd('devtool modify %s -x %s' % (testrecipe, tempsrcdir))
# Check git repo
- self.assertTrue(os.path.isdir(os.path.join(tempsrcdir, '.git')), 'git repository for external source tree not found')
- result = runCmd('git status --porcelain', cwd=tempsrcdir)
- self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')
- result = runCmd('git symbolic-ref HEAD', cwd=tempsrcdir)
- self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')
+ self._check_src_repo(tempsrcdir)
# Add a commit
result = runCmd('echo "# Additional line" >> Makefile', cwd=tempsrcdir)
result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir)
@@ -778,7 +765,7 @@ class DevtoolTests(DevtoolBase):
self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
result = runCmd('devtool extract remake %s' % tempdir)
self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found')
- self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')
+ self._check_src_repo(tempdir)
@testcase(1168)
def test_devtool_reset_all(self):
--
2.1.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH v3 05/10] oe-selftest: devtool: add method for checking repo status
2015-09-24 11:52 [PATCH v3 00/10] devtool: improve handling of local source files Markus Lehtonen
` (3 preceding siblings ...)
2015-09-24 11:53 ` [PATCH v3 04/10] oe-selftest: devtool: add method for checking srctree repo Markus Lehtonen
@ 2015-09-24 11:53 ` Markus Lehtonen
2015-09-24 11:53 ` [PATCH v3 06/10] devtool: update-recipe: add new patches in correct order Markus Lehtonen
` (4 subsequent siblings)
9 siblings, 0 replies; 18+ messages in thread
From: Markus Lehtonen @ 2015-09-24 11:53 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
New method for checking the status of the working tree of a repository.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/selftest/devtool.py | 87 ++++++++++++++++-----------------------
1 file changed, 35 insertions(+), 52 deletions(-)
diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
index f459a6d..a893ed3 100644
--- a/meta/lib/oeqa/selftest/devtool.py
+++ b/meta/lib/oeqa/selftest/devtool.py
@@ -103,6 +103,23 @@ class DevtoolTests(DevtoolBase):
self.assertEqual(result.output.strip(), "refs/heads/devtool",
'Wrong branch in git repo')
+ def _check_repo_status(self, repo_dir, expected_status):
+ """Check the worktree status of a repository"""
+ result = runCmd('git status . --porcelain',
+ cwd=repo_dir)
+ for line in result.output.splitlines():
+ for ind, (f_status, fn_re) in enumerate(expected_status):
+ if re.match(fn_re, line[3:]):
+ if f_status != line[:2]:
+ self.fail('Unexpected status in line: %s' % line)
+ expected_status.pop(ind)
+ break
+ else:
+ self.fail('Unexpected modified file in line: %s' % line)
+ if expected_status:
+ self.fail('Missing file changes: %s' % expected_status)
+
+
@testcase(1158)
def test_create_workspace(self):
# Check preconditions
@@ -468,8 +485,7 @@ class DevtoolTests(DevtoolBase):
recipefile = get_bb_var('FILE', testrecipe)
src_uri = get_bb_var('SRC_URI', testrecipe)
self.assertNotIn('git://', src_uri, 'This test expects the %s recipe to NOT be a git recipe' % testrecipe)
- result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))
- self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)
+ self._check_repo_status(os.path.dirname(recipefile), [])
# First, modify a recipe
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
self.track_for_cleanup(tempdir)
@@ -488,19 +504,10 @@ class DevtoolTests(DevtoolBase):
result = runCmd('git commit -m "Add a new file"', cwd=tempdir)
self.add_command_to_tearDown('cd %s; rm %s/*.patch; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))
result = runCmd('devtool update-recipe %s' % testrecipe)
- result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))
- self.assertNotEqual(result.output.strip(), "", '%s recipe should be modified' % testrecipe)
- status = result.output.splitlines()
- self.assertEqual(len(status), 3, 'Less/more files modified than expected. Entire status:\n%s' % result.output)
- for line in status:
- if line.endswith('0001-Change-the-README.patch'):
- self.assertEqual(line[:3], '?? ', 'Unexpected status in line: %s' % line)
- elif line.endswith('0002-Add-a-new-file.patch'):
- self.assertEqual(line[:3], '?? ', 'Unexpected status in line: %s' % line)
- elif re.search('%s_[^_]*.bb$' % testrecipe, line):
- self.assertEqual(line[:3], ' M ', 'Unexpected status in line: %s' % line)
- else:
- raise AssertionError('Unexpected modified file in status: %s' % line)
+ expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)),
+ ('??', '.*/0001-Change-the-README.patch$'),
+ ('??', '.*/0002-Add-a-new-file.patch$')]
+ self._check_repo_status(os.path.dirname(recipefile), expected_status)
@testcase(1172)
def test_devtool_update_recipe_git(self):
@@ -515,8 +522,7 @@ class DevtoolTests(DevtoolBase):
if entry.startswith('file://') and entry.endswith('.patch'):
patches.append(entry[7:].split(';')[0])
self.assertGreater(len(patches), 0, 'The %s recipe does not appear to contain any patches, so this test will not be effective' % testrecipe)
- result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))
- self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)
+ self._check_repo_status(os.path.dirname(recipefile), [])
# First, modify a recipe
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
self.track_for_cleanup(tempdir)
@@ -535,19 +541,10 @@ class DevtoolTests(DevtoolBase):
result = runCmd('git commit -m "Add a new file"', cwd=tempdir)
self.add_command_to_tearDown('cd %s; rm -rf %s; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))
result = runCmd('devtool update-recipe -m srcrev %s' % testrecipe)
- result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))
- self.assertNotEqual(result.output.strip(), "", '%s recipe should be modified' % testrecipe)
- status = result.output.splitlines()
- for line in status:
- for patch in patches:
- if line.endswith(patch):
- self.assertEqual(line[:3], ' D ', 'Unexpected status in line: %s' % line)
- break
- else:
- if re.search('%s_[^_]*.bb$' % testrecipe, line):
- self.assertEqual(line[:3], ' M ', 'Unexpected status in line: %s' % line)
- else:
- raise AssertionError('Unexpected modified file in status: %s' % line)
+ expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile))] + \
+ [(' D', '.*/%s$' % patch) for patch in patches]
+ self._check_repo_status(os.path.dirname(recipefile), expected_status)
+
result = runCmd('git diff %s' % os.path.basename(recipefile), cwd=os.path.dirname(recipefile))
addlines = ['SRCREV = ".*"', 'SRC_URI = "git://git.infradead.org/mtd-utils.git"']
srcurilines = src_uri.split()
@@ -576,21 +573,11 @@ class DevtoolTests(DevtoolBase):
result = runCmd('devtool update-recipe %s' % testrecipe)
result = runCmd('git rev-parse --show-toplevel')
topleveldir = result.output.strip()
- result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))
- status = result.output.splitlines()
relpatchpath = os.path.join(os.path.relpath(os.path.dirname(recipefile), topleveldir), testrecipe)
- expectedstatus = [('M', os.path.relpath(recipefile, topleveldir)),
- ('??', '%s/0001-Change-the-Makefile.patch' % relpatchpath),
- ('??', '%s/0002-Add-a-new-file.patch' % relpatchpath)]
- for line in status:
- statusline = line.split(None, 1)
- for fstatus, fn in expectedstatus:
- if fn == statusline[1]:
- if fstatus != statusline[0]:
- self.fail('Unexpected status in line: %s' % line)
- break
- else:
- self.fail('Unexpected modified file in line: %s' % line)
+ expected_status = [(' M', os.path.relpath(recipefile, topleveldir)),
+ ('??', '%s/0001-Change-the-Makefile.patch' % relpatchpath),
+ ('??', '%s/0002-Add-a-new-file.patch' % relpatchpath)]
+ self._check_repo_status(os.path.dirname(recipefile), expected_status)
@testcase(1170)
def test_devtool_update_recipe_append(self):
@@ -600,8 +587,7 @@ class DevtoolTests(DevtoolBase):
recipefile = get_bb_var('FILE', testrecipe)
src_uri = get_bb_var('SRC_URI', testrecipe)
self.assertNotIn('git://', src_uri, 'This test expects the %s recipe to NOT be a git recipe' % testrecipe)
- result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))
- self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)
+ self._check_repo_status(os.path.dirname(recipefile), [])
# First, modify a recipe
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
tempsrcdir = os.path.join(tempdir, 'source')
@@ -623,8 +609,7 @@ class DevtoolTests(DevtoolBase):
result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))
self.assertNotIn('WARNING:', result.output)
# Check recipe is still clean
- result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))
- self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)
+ self._check_repo_status(os.path.dirname(recipefile), [])
# Check bbappend was created
splitpath = os.path.dirname(recipefile).split(os.sep)
appenddir = os.path.join(templayerdir, splitpath[-2], splitpath[-1])
@@ -675,8 +660,7 @@ class DevtoolTests(DevtoolBase):
if entry.startswith('git://'):
git_uri = entry
break
- result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))
- self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)
+ self._check_repo_status(os.path.dirname(recipefile), [])
# First, modify a recipe
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
tempsrcdir = os.path.join(tempdir, 'source')
@@ -707,8 +691,7 @@ class DevtoolTests(DevtoolBase):
result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir))
self.assertNotIn('WARNING:', result.output)
# Check recipe is still clean
- result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))
- self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)
+ self._check_repo_status(os.path.dirname(recipefile), [])
# Check bbappend was created
splitpath = os.path.dirname(recipefile).split(os.sep)
appenddir = os.path.join(templayerdir, splitpath[-2], splitpath[-1])
--
2.1.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH v3 06/10] devtool: update-recipe: add new patches in correct order
2015-09-24 11:52 [PATCH v3 00/10] devtool: improve handling of local source files Markus Lehtonen
` (4 preceding siblings ...)
2015-09-24 11:53 ` [PATCH v3 05/10] oe-selftest: devtool: add method for checking repo status Markus Lehtonen
@ 2015-09-24 11:53 ` Markus Lehtonen
2015-09-24 11:53 ` [PATCH v3 07/10] devtool: update_recipe: refactor patch generation Markus Lehtonen
` (3 subsequent siblings)
9 siblings, 0 replies; 18+ messages in thread
From: Markus Lehtonen @ 2015-09-24 11:53 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
When adding multiple new patches append them to SRC_URI in correct order
so that they apply correctly.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
scripts/lib/devtool/standard.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 96b271c..1154030 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -701,7 +701,7 @@ def _update_recipe_patch(args, config, srctree, rd, config_data):
updatepatches = False
updaterecipe = False
destpath = None
- newpatches = os.listdir(tempdir)
+ newpatches = sorted(os.listdir(tempdir))
if args.append:
patchfiles = {}
for patch in existing_patches:
--
2.1.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH v3 07/10] devtool: update_recipe: refactor patch generation
2015-09-24 11:52 [PATCH v3 00/10] devtool: improve handling of local source files Markus Lehtonen
` (5 preceding siblings ...)
2015-09-24 11:53 ` [PATCH v3 06/10] devtool: update-recipe: add new patches in correct order Markus Lehtonen
@ 2015-09-24 11:53 ` Markus Lehtonen
2015-09-24 11:53 ` [PATCH v3 08/10] devtool: file mover function that creates target dir Markus Lehtonen
` (2 subsequent siblings)
9 siblings, 0 replies; 18+ messages in thread
From: Markus Lehtonen @ 2015-09-24 11:53 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
Implement new function that handles patch file generation. The new
function also does the discovery of new, updated and deleted patches.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
scripts/lib/devtool/standard.py | 119 ++++++++++++++++++++++++----------------
1 file changed, 72 insertions(+), 47 deletions(-)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 1154030..7c8e447 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -25,6 +25,7 @@ import logging
import argparse
import scriptutils
import errno
+from collections import OrderedDict
from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, DevtoolError
from devtool import parse_recipe
@@ -590,11 +591,55 @@ def _remove_patch_files(args, patches, destpath):
if ose.errno != errno.ENOTEMPTY:
raise
+
+def _export_patches(srctree, rd, start_rev, destdir):
+ """Export patches from srctree to given location.
+ Returns three-tuple of dicts:
+ 1. updated - patches that already exist in SRCURI
+ 2. added - new patches that don't exist in SRCURI
+ 3 removed - patches that exist in SRCURI but not in exported patches
+ In each dict the key is the 'basepath' of the URI and value is the
+ absolute path to the existing file in recipe space (if any).
+ """
+ import oe.recipeutils
+ from oe.patch import GitApplyTree
+ updated = OrderedDict()
+ added = OrderedDict()
+ seqpatch_re = re.compile('^([0-9]{4}-)?(.+)')
+
+ existing_patches = dict((os.path.basename(path), path) for path in
+ oe.recipeutils.get_recipe_patches(rd))
+
+ # Generate patches from Git
+ GitApplyTree.extractPatches(srctree, start_rev, destdir)
+
+ new_patches = sorted(os.listdir(destdir))
+ for new_patch in new_patches:
+ # Strip numbering from patch names. If it's a git sequence named patch,
+ # the numbers might not match up since we are starting from a different
+ # revision This does assume that people are using unique shortlog
+ # values, but they ought to be anyway...
+ new_basename = seqpatch_re.match(new_patch).group(2)
+ found = False
+ for old_patch in existing_patches:
+ old_basename = seqpatch_re.match(old_patch).group(2)
+ if new_basename == old_basename:
+ updated[new_patch] = existing_patches.pop(old_patch)
+ found = True
+ # Rename patch files
+ if new_patch != old_patch:
+ os.rename(os.path.join(destdir, new_patch),
+ os.path.join(destdir, old_patch))
+ break
+ if not found:
+ added[new_patch] = None
+ return (updated, added, existing_patches)
+
+
def _update_recipe_srcrev(args, srctree, rd, config_data):
"""Implement the 'srcrev' mode of update-recipe"""
import bb
import oe.recipeutils
- from oe.patch import GitApplyTree
recipefile = rd.getVar('FILE', True)
logger.info('Updating SRCREV in recipe %s' % os.path.basename(recipefile))
@@ -621,12 +666,10 @@ def _update_recipe_srcrev(args, srctree, rd, config_data):
old_srcrev = (rd.getVar('SRCREV', False) or '')
tempdir = tempfile.mkdtemp(prefix='devtool')
try:
- GitApplyTree.extractPatches(srctree, old_srcrev, tempdir)
- newpatches = os.listdir(tempdir)
- for patch in existing_patches:
- patchfile = os.path.basename(patch)
- if patchfile in newpatches:
- removepatches.append(patch)
+ upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev,
+ tempdir)
+ # Remove "overlapping" patches
+ removepatches = upd_p.values()
finally:
shutil.rmtree(tempdir)
@@ -654,7 +697,6 @@ def _update_recipe_patch(args, config, srctree, rd, config_data):
"""Implement the 'patch' mode of update-recipe"""
import bb
import oe.recipeutils
- from oe.patch import GitApplyTree
recipefile = rd.getVar('FILE', True)
append = os.path.join(config.workspace_path, 'appends', '%s.bbappend' %
@@ -677,40 +719,27 @@ def _update_recipe_patch(args, config, srctree, rd, config_data):
# Get all patches from source tree and check if any should be removed
tempdir = tempfile.mkdtemp(prefix='devtool')
try:
- GitApplyTree.extractPatches(srctree, initial_rev, tempdir)
- # Strip numbering from patch names. If it's a git sequence named
- # patch, the numbers might not match up since we are starting from
- # a different revision This does assume that people are using
- # unique shortlog values, but they ought to be anyway...
- newpatches = [seqpatch_re.match(fname).group(2) for fname in
- os.listdir(tempdir)]
- for patch in existing_patches:
- basename = seqpatch_re.match(
- os.path.basename(patch)).group(2)
- if basename not in newpatches:
- removepatches.append(patch)
+ # Get all patches from source tree and check if any should be removed
+ upd_p, new_p, del_p = _export_patches(srctree, rd, initial_rev,
+ tempdir)
+ # Remove deleted patches
+ removepatches = del_p.values()
finally:
shutil.rmtree(tempdir)
# Get updated patches from source tree
tempdir = tempfile.mkdtemp(prefix='devtool')
try:
- GitApplyTree.extractPatches(srctree, update_rev, tempdir)
+ upd_p, new_p, del_p = _export_patches(srctree, rd, update_rev,
+ tempdir)
# Match up and replace existing patches with corresponding new patches
updatepatches = False
updaterecipe = False
destpath = None
- newpatches = sorted(os.listdir(tempdir))
if args.append:
- patchfiles = {}
- for patch in existing_patches:
- patchfile = os.path.basename(patch)
- if patchfile in newpatches:
- patchfiles[os.path.join(tempdir, patchfile)] = patchfile
- newpatches.remove(patchfile)
- for patchfile in newpatches:
- patchfiles[os.path.join(tempdir, patchfile)] = None
+ patchfiles = dict((os.path.join(tempdir, key), val) for
+ key, val in upd_p.items() + new_p.items())
if patchfiles or removepatches:
removevalues = None
@@ -728,25 +757,21 @@ def _update_recipe_patch(args, config, srctree, rd, config_data):
else:
logger.info('No patches needed updating')
else:
- for patch in existing_patches:
- patchfile = os.path.basename(patch)
- if patchfile in newpatches:
- logger.info('Updating patch %s' % patchfile)
- shutil.move(os.path.join(tempdir, patchfile), patch)
- newpatches.remove(patchfile)
- updatepatches = True
+ for basepath, path in upd_p.iteritems():
+ logger.info('Updating patch %s' % basepath)
+ shutil.move(os.path.join(tempdir, basepath), path)
+ updatepatches = True
srcuri = (rd.getVar('SRC_URI', False) or '').split()
- if newpatches:
- # Add any patches left over
- patchdir = os.path.join(os.path.dirname(recipefile),
- rd.getVar('BPN', True))
+ patchdir = os.path.join(os.path.dirname(recipefile),
+ rd.getVar('BPN', True))
+ bb.utils.mkdirhier(patchdir)
+ for basepath, path in new_p.iteritems():
+ logger.info('Adding new patch %s' % basepath)
bb.utils.mkdirhier(patchdir)
- for patchfile in newpatches:
- logger.info('Adding new patch %s' % patchfile)
- shutil.move(os.path.join(tempdir, patchfile),
- os.path.join(patchdir, patchfile))
- srcuri.append('file://%s' % patchfile)
- updaterecipe = True
+ shutil.move(os.path.join(tempdir, basepath),
+ os.path.join(patchdir, basepath))
+ srcuri.append('file://%s' % basepath)
+ updaterecipe = True
if removepatches:
removedentries, _ = _remove_patch_entries(srcuri, removepatches)
if removedentries:
--
2.1.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH v3 08/10] devtool: file mover function that creates target dir
2015-09-24 11:52 [PATCH v3 00/10] devtool: improve handling of local source files Markus Lehtonen
` (6 preceding siblings ...)
2015-09-24 11:53 ` [PATCH v3 07/10] devtool: update_recipe: refactor patch generation Markus Lehtonen
@ 2015-09-24 11:53 ` Markus Lehtonen
2015-09-24 11:53 ` [PATCH v3 09/10] devtool: better support for local source files Markus Lehtonen
2015-09-24 11:53 ` [PATCH v3 10/10] devtool: modify: make bitbake use local files from srctree Markus Lehtonen
9 siblings, 0 replies; 18+ messages in thread
From: Markus Lehtonen @ 2015-09-24 11:53 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
Helper function for replacing a pattern like:
target_dir = os.path.dirname(target)
bb.utils.mkdirhier(target_dir)
shutil.move(source, target)
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
scripts/lib/devtool/standard.py | 16 +++++++++++-----
1 file changed, 11 insertions(+), 5 deletions(-)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 7c8e447..efa6fd1 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -174,6 +174,13 @@ def _check_compatible_recipe(pn, d):
"from working. You will need to disable this "
"first." % pn)
+def _move_file(src, dst):
+ """Move a file. Creates all the directory components of destination path."""
+ dst_d = os.path.dirname(dst)
+ if dst_d:
+ bb.utils.mkdirhier(dst_d)
+ shutil.move(src, dst)
+
def _ls_tree(directory):
"""Recursive listing of files in a directory"""
ret = []
@@ -330,9 +337,8 @@ def _extract_source(srctree, keep_temp, devbranch, d):
crd.setVar('S', srcsubdir)
# Move source files to S
for path in src_files:
- tgt_dir = os.path.join(srcsubdir, os.path.dirname(path))
- bb.utils.mkdirhier(tgt_dir)
- shutil.move(os.path.join(workdir, path), tgt_dir)
+ _move_file(os.path.join(workdir, path),
+ os.path.join(srcsubdir, path))
elif os.path.dirname(srcsubdir) != workdir:
# Handle if S is set to a subdirectory of the source
srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
@@ -893,8 +899,8 @@ def reset(args, config, basepath, workspace):
for root, dirs, files in os.walk(origdir):
for fn in files:
logger.warn('Preserving %s in %s' % (fn, preservepath))
- bb.utils.mkdirhier(preservepath)
- shutil.move(os.path.join(origdir, fn), os.path.join(preservepath, fn))
+ _move_file(os.path.join(origdir, fn),
+ os.path.join(preservepath, fn))
for dn in dirs:
os.rmdir(os.path.join(root, dn))
os.rmdir(origdir)
--
2.1.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH v3 09/10] devtool: better support for local source files
2015-09-24 11:52 [PATCH v3 00/10] devtool: improve handling of local source files Markus Lehtonen
` (7 preceding siblings ...)
2015-09-24 11:53 ` [PATCH v3 08/10] devtool: file mover function that creates target dir Markus Lehtonen
@ 2015-09-24 11:53 ` Markus Lehtonen
2015-09-24 11:53 ` [PATCH v3 10/10] devtool: modify: make bitbake use local files from srctree Markus Lehtonen
9 siblings, 0 replies; 18+ messages in thread
From: Markus Lehtonen @ 2015-09-24 11:53 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
* extract: Copy all local source files (i.e. non-compressed/non-arcived
SRC_URI files that have file:// URI prefix) - excluding patches - to
the srctree repository. The files will be placed in a subdirectory
called 'oe-local-files'. The oe-local-files directory is not committed
to the Git repository, but, marked to be ignored by a .gitignore file.
The developer can manually add and commit the files to Git if the
changes to them need to be tracked.
Before this patch, local source files (were copied (and committed) to
the srctree repository only in some special cases (basically when
S=WORKDIR) when doing devtool-extract. For most of the packages local
files were not copied at all.
* update-recipe: This patch causes the local files to be 'synced' from
the srctree (i.e. from the 'oe-local-files' subdirectory) to the
layer. Being 'synced' means that in addition to copying modified
files over the original sources, devtool will also handle removing and
adding local source files and updating the recipe accordingly. We
don't want to create patches against the local source files but rather
update them directly. Thus, 'oe-local-file' directory is ignored in
patch generation when doing update-recipe, even if committed to Git.
This functionality is only enabled if the 'oe-local-files' directory
is present in srctree.
[YOCTO #7602]
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/selftest/devtool.py | 73 +++++++++
scripts/lib/devtool/__init__.py | 10 +-
scripts/lib/devtool/standard.py | 314 ++++++++++++++++++++++++++------------
3 files changed, 299 insertions(+), 98 deletions(-)
diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
index a893ed3..59f0fae 100644
--- a/meta/lib/oeqa/selftest/devtool.py
+++ b/meta/lib/oeqa/selftest/devtool.py
@@ -738,6 +738,79 @@ class DevtoolTests(DevtoolBase):
self.assertEqual(expectedlines, f.readlines())
# Deleting isn't expected to work under these circumstances
+ @testcase(1173)
+ def test_devtool_update_recipe_local_files(self):
+ """Check that local source files are copied over instead of patched"""
+ workspacedir = self._get_workspace_dir()
+ testrecipe = 'makedevs'
+ recipefile = get_bb_var('FILE', testrecipe)
+ # Setup srctree for modifying the recipe
+ tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+ self.track_for_cleanup(tempdir)
+ self.track_for_cleanup(workspacedir)
+ self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+ # (don't bother with cleaning the recipe on teardown, we won't be
+ # building it)
+ result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
+ # Check git repo
+ self._check_src_repo(tempdir)
+ # Edit / commit local source
+ runCmd('echo "/* Foobar */" >> oe-local-files/makedevs.c', cwd=tempdir)
+ runCmd('echo "Foo" > oe-local-files/new-local', cwd=tempdir)
+ runCmd('echo "Bar" > new-file', cwd=tempdir)
+ runCmd('git add new-file', cwd=tempdir)
+ runCmd('git commit -m "Add new file"', cwd=tempdir)
+ self.add_command_to_tearDown('cd %s; git clean -fd .; git checkout .' %
+ os.path.dirname(recipefile))
+ runCmd('devtool update-recipe %s' % testrecipe)
+ expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)),
+ (' M', '.*/makedevs/makedevs.c$'),
+ ('??', '.*/makedevs/new-local$'),
+ ('??', '.*/makedevs/0001-Add-new-file.patch$')]
+ self._check_repo_status(os.path.dirname(recipefile), expected_status)
+
+ @testcase(1174)
+ def test_devtool_update_recipe_local_files_2(self):
+ """Check local source files support when oe-local-files is in Git"""
+ workspacedir = self._get_workspace_dir()
+ testrecipe = 'lzo'
+ recipefile = get_bb_var('FILE', testrecipe)
+ # Setup srctree for modifying the recipe
+ tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+ self.track_for_cleanup(tempdir)
+ self.track_for_cleanup(workspacedir)
+ self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+ result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
+ # Check git repo
+ self._check_src_repo(tempdir)
+ # Add oe-local-files to Git
+ runCmd('rm oe-local-files/.gitignore', cwd=tempdir)
+ runCmd('git add oe-local-files', cwd=tempdir)
+ runCmd('git commit -m "Add local sources"', cwd=tempdir)
+ # Edit / commit local sources
+ runCmd('echo "# Foobar" >> oe-local-files/acinclude.m4', cwd=tempdir)
+ runCmd('git commit -am "Edit existing file"', cwd=tempdir)
+ runCmd('git rm oe-local-files/run-ptest', cwd=tempdir)
+ runCmd('git commit -m"Remove file"', cwd=tempdir)
+ runCmd('echo "Foo" > oe-local-files/new-local', cwd=tempdir)
+ runCmd('git add oe-local-files/new-local', cwd=tempdir)
+ runCmd('git commit -m "Add new local file"', cwd=tempdir)
+ runCmd('echo "Gar" > new-file', cwd=tempdir)
+ runCmd('git add new-file', cwd=tempdir)
+ runCmd('git commit -m "Add new file"', cwd=tempdir)
+ self.add_command_to_tearDown('cd %s; git clean -fd .; git checkout .' %
+ os.path.dirname(recipefile))
+ # Checkout unmodified file to working copy -> devtool should still pick
+ # the modified version from HEAD
+ runCmd('git checkout HEAD^ -- oe-local-files/acinclude.m4', cwd=tempdir)
+ runCmd('devtool update-recipe %s' % testrecipe)
+ expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)),
+ (' M', '.*/acinclude.m4$'),
+ (' D', '.*/run-ptest$'),
+ ('??', '.*/new-local$'),
+ ('??', '.*/0001-Add-new-file.patch$')]
+ self._check_repo_status(os.path.dirname(recipefile), expected_status)
+
@testcase(1163)
def test_devtool_extract(self):
workspacedir = self._get_workspace_dir()
diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py
index f815ef2..54792c2 100644
--- a/scripts/lib/devtool/__init__.py
+++ b/scripts/lib/devtool/__init__.py
@@ -179,11 +179,17 @@ def setup_git_repo(repodir, version, devbranch, basetag='devtool-base'):
if not os.path.exists(os.path.join(repodir, '.git')):
bb.process.run('git init', cwd=repodir)
bb.process.run('git add .', cwd=repodir)
- if version:
+ commit_cmd = ['git', 'commit', '-q']
+ stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
+ if not stdout:
+ commit_cmd.append('--allow-empty')
+ commitmsg = "Initial empty commit with no upstream sources"
+ elif version:
commitmsg = "Initial commit from upstream at version %s" % version
else:
commitmsg = "Initial commit from upstream"
- bb.process.run('git commit -q -m "%s"' % commitmsg, cwd=repodir)
+ commit_cmd += ['-m', commitmsg]
+ bb.process.run(commit_cmd, cwd=repodir)
bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
bb.process.run('git tag -f %s' % basetag, cwd=repodir)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index efa6fd1..6b85c8c 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -181,6 +181,36 @@ def _move_file(src, dst):
bb.utils.mkdirhier(dst_d)
shutil.move(src, dst)
+def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
+ """List contents of a git treeish"""
+ import bb
+ cmd = ['git', 'ls-tree', '-z', treeish]
+ if recursive:
+ cmd.append('-r')
+ out, _ = bb.process.run(cmd, cwd=repodir)
+ ret = {}
+ for line in out.split('\0'):
+ if line:
+ split = line.split(None, 4)
+ ret[split[3]] = split[0:3]
+ return ret
+
+def _git_exclude_path(srctree, path):
+ """Return pathspec (list of paths) that excludes certain path"""
+ # NOTE: "Filtering out" files/paths in this way is not entirely reliable -
+ # we don't catch files that are deleted, for example. A more reliable way
+ # to implement this would be to use "negative pathspecs" which were
+ # introduced in Git v1.9.0. Revisit this when/if the required Git version
+ # becomes greater than that.
+ path = os.path.normpath(path)
+ recurse = True if len(path.split(os.path.sep)) > 1 else False
+ git_files = _git_ls_tree(srctree, 'HEAD', recurse).keys()
+ if path in git_files:
+ git_files.remove(path)
+ return git_files
+ else:
+ return ['.']
+
def _ls_tree(directory):
"""Recursive listing of files in a directory"""
ret = []
@@ -326,10 +356,25 @@ def _extract_source(srctree, keep_temp, devbranch, d):
logger.info('Doing kernel checkout...')
task_executor.exec_func('do_kernel_checkout', False)
srcsubdir = crd.getVar('S', True)
+
+ # Move local source files into separate subdir
+ recipe_patches = [os.path.basename(patch) for patch in
+ oe.recipeutils.get_recipe_patches(crd)]
+ local_files = oe.recipeutils.get_recipe_local_files(crd)
+ local_files = [fname for fname in local_files if
+ os.path.exists(os.path.join(workdir, fname))]
+ if local_files:
+ for fname in local_files:
+ _move_file(os.path.join(workdir, fname),
+ os.path.join(tempdir, 'oe-local-files', fname))
+ with open(os.path.join(tempdir, 'oe-local-files', '.gitignore'),
+ 'w') as f:
+ f.write('# Ignore local files, by default. Remove this file '
+ 'if you want to commit the directory to Git\n*\n')
+
if srcsubdir == workdir:
- # Find non-patch sources that were "unpacked" to srctree directory
- recipe_patches = [os.path.basename(patch) for patch in
- oe.recipeutils.get_recipe_patches(crd)]
+ # Find non-patch non-local sources that were "unpacked" to srctree
+ # directory
src_files = [fname for fname in _ls_tree(workdir) if
os.path.basename(fname) not in recipe_patches]
# Force separate S so that patch files can be left out from srctree
@@ -352,12 +397,12 @@ def _extract_source(srctree, keep_temp, devbranch, d):
haspatches = True
else:
os.rmdir(patchdir)
-
+ # Make sure that srcsubdir exists
+ bb.utils.mkdirhier(srcsubdir)
if not os.path.exists(srcsubdir) or not os.listdir(srcsubdir):
- raise DevtoolError("no source unpacked to S, either the %s "
- "recipe doesn't use any source or the "
- "correct source directory could not be "
- "determined" % pn)
+ logger.warning("no source unpacked to S, either the %s recipe "
+ "doesn't use any source or the correct source "
+ "directory could not be determined" % pn)
setup_git_repo(srcsubdir, crd.getVar('PV', True), devbranch)
@@ -376,6 +421,12 @@ def _extract_source(srctree, keep_temp, devbranch, d):
if haspatches:
bb.process.run('git checkout patches', cwd=srcsubdir)
+ # Move oe-local-files directory to srctree
+ if os.path.exists(os.path.join(tempdir, 'oe-local-files')):
+ logger.info('Adding local source files to srctree...')
+ shutil.move(os.path.join(tempdir, 'oe-local-files'), srcsubdir)
+
+
shutil.move(srcsubdir, srctree)
finally:
bb.logger.setLevel(origlevel)
@@ -560,39 +611,40 @@ def _get_patchset_revs(args, srctree, recipe_path):
return initial_rev, update_rev
-def _remove_patch_entries(srcuri, patchlist):
- """Remove patch entries from SRC_URI"""
- remaining = patchlist[:]
+def _remove_file_entries(srcuri, filelist):
+ """Remove file:// entries from SRC_URI"""
+ remaining = filelist[:]
entries = []
- for patch in patchlist:
- patchfile = os.path.basename(patch)
+ for fname in filelist:
+ basename = os.path.basename(fname)
for i in xrange(len(srcuri)):
- if srcuri[i].startswith('file://') and os.path.basename(srcuri[i].split(';')[0]) == patchfile:
+ if (srcuri[i].startswith('file://') and
+ os.path.basename(srcuri[i].split(';')[0]) == basename):
entries.append(srcuri[i])
- remaining.remove(patch)
+ remaining.remove(fname)
srcuri.pop(i)
break
return entries, remaining
-def _remove_patch_files(args, patches, destpath):
+def _remove_source_files(args, files, destpath):
"""Unlink existing patch files"""
- for patchfile in patches:
+ for path in files:
if args.append:
if not destpath:
raise Exception('destpath should be set here')
- patchfile = os.path.join(destpath, os.path.basename(patchfile))
+ path = os.path.join(destpath, os.path.basename(path))
- if os.path.exists(patchfile):
- logger.info('Removing patch %s' % patchfile)
+ if os.path.exists(path):
+ logger.info('Removing file %s' % path)
# FIXME "git rm" here would be nice if the file in question is
# tracked
# FIXME there's a chance that this file is referred to by
# another recipe, in which case deleting wouldn't be the
# right thing to do
- os.remove(patchfile)
+ os.remove(path)
# Remove directory if empty
try:
- os.rmdir(os.path.dirname(patchfile))
+ os.rmdir(os.path.dirname(path))
except OSError as ose:
if ose.errno != errno.ENOTEMPTY:
raise
@@ -616,8 +668,9 @@ def _export_patches(srctree, rd, start_rev, destdir):
existing_patches = dict((os.path.basename(path), path) for path in
oe.recipeutils.get_recipe_patches(rd))
- # Generate patches from Git
- GitApplyTree.extractPatches(srctree, start_rev, destdir)
+ # Generate patches from Git, exclude local files directory
+ patch_pathspec = _git_exclude_path(srctree, 'oe-local-files')
+ GitApplyTree.extractPatches(srctree, start_rev, destdir, patch_pathspec)
new_patches = sorted(os.listdir(destdir))
for new_patch in new_patches:
@@ -642,6 +695,52 @@ def _export_patches(srctree, rd, start_rev, destdir):
return (updated, added, existing_patches)
+def _export_local_files(srctree, rd, destdir):
+ """Copy local files from srctree to given location.
+ Returns three-tuple of dicts:
+ 1. updated - files that already exist in SRCURI
+ 2. added - new files files that don't exist in SRCURI
+ 3 removed - files that exist in SRCURI but not in exported files
+ In each dict the key is the 'basepath' of the URI and value is the
+ absolute path to the existing file in recipe space (if any).
+ """
+ import oe.recipeutils
+
+ # Find out local files (SRC_URI files that exist in the "recipe space").
+ # Local files that reside in srctree are not included in patch generation.
+ # Instead they are directly copied over the original source files (in
+ # recipe space).
+ existing_files = oe.recipeutils.get_recipe_local_files(rd)
+ new_set = None
+ updated = OrderedDict()
+ added = OrderedDict()
+ removed = OrderedDict()
+ git_files = _git_ls_tree(srctree)
+ if 'oe-local-files' in git_files:
+ # If tracked by Git, take the files from srctree HEAD. First get
+ # the tree object of the directory
+ tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool')
+ tree = git_files['oe-local-files'][2]
+ bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree,
+ env=dict(os.environ, GIT_WORK_TREE=destdir,
+ GIT_INDEX_FILE=tmp_index))
+ new_set = _git_ls_tree(srctree, tree, True).keys()
+ elif os.path.isdir(os.path.join(srctree, 'oe-local-files')):
+ # If not tracked by Git, just copy from working copy
+ new_set = _ls_tree(os.path.join(srctree, 'oe-local-files'))
+ bb.process.run(['cp', '-ax',
+ os.path.join(srctree, 'oe-local-files', '.'), destdir])
+ if new_set is not None:
+ for fname in new_set:
+ if fname in existing_files:
+ updated[fname] = existing_files.pop(fname)
+ elif fname != '.gitignore':
+ added[fname] = None
+
+ removed = existing_files
+ return (updated, added, removed)
+
+
def _update_recipe_srcrev(args, srctree, rd, config_data):
"""Implement the 'srcrev' mode of update-recipe"""
import bb
@@ -661,43 +760,63 @@ def _update_recipe_srcrev(args, srctree, rd, config_data):
raise DevtoolError('Invalid hash returned by git: %s' % stdout)
destpath = None
- removepatches = []
+ remove_files = []
patchfields = {}
patchfields['SRCREV'] = srcrev
orig_src_uri = rd.getVar('SRC_URI', False) or ''
- if not args.no_remove:
- # Find list of existing patches in recipe file
- existing_patches = oe.recipeutils.get_recipe_patches(rd)
-
- old_srcrev = (rd.getVar('SRCREV', False) or '')
- tempdir = tempfile.mkdtemp(prefix='devtool')
- try:
+ srcuri = orig_src_uri.split()
+ tempdir = tempfile.mkdtemp(prefix='devtool')
+ update_srcuri = False
+ try:
+ local_files_dir = tempfile.mkdtemp(dir=tempdir)
+ upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir)
+ if not args.no_remove:
+ # Find list of existing patches in recipe file
+ patches_dir = tempfile.mkdtemp(dir=tempdir)
+ old_srcrev = (rd.getVar('SRCREV', False) or '')
upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev,
- tempdir)
- # Remove "overlapping" patches
- removepatches = upd_p.values()
- finally:
- shutil.rmtree(tempdir)
-
- if removepatches:
- srcuri = orig_src_uri.split()
- removedentries, _ = _remove_patch_entries(srcuri, removepatches)
- if removedentries:
- patchfields['SRC_URI'] = ' '.join(srcuri)
+ patches_dir)
- if args.append:
- _, destpath = oe.recipeutils.bbappend_recipe(
- rd, args.append, None, wildcardver=args.wildcard_version,
- extralines=patchfields)
- else:
- oe.recipeutils.patch_recipe(rd, recipefile, patchfields)
+ # Remove deleted local files and "overlapping" patches
+ remove_files = del_f.values() + upd_p.values()
+ if remove_files:
+ removedentries = _remove_file_entries(srcuri, remove_files)[0]
+ update_srcuri = True
+ if args.append:
+ files = dict((os.path.join(local_files_dir, key), val) for
+ key, val in upd_f.items() + new_f.items())
+ removevalues = {}
+ if update_srcuri:
+ removevalues = {'SRC_URI': removedentries}
+ patchfields['SRC_URI'] = '\\\n '.join(srcuri)
+ _, destpath = oe.recipeutils.bbappend_recipe(
+ rd, args.append, files, wildcardver=args.wildcard_version,
+ extralines=patchfields, removevalues=removevalues)
+ else:
+ files_dir = os.path.join(os.path.dirname(recipefile),
+ rd.getVar('BPN', True))
+ for basepath, path in upd_f.iteritems():
+ logger.info('Updating file %s' % basepath)
+ _move_file(os.path.join(local_files_dir, basepath), path)
+ update_srcuri= True
+ for basepath, path in new_f.iteritems():
+ logger.info('Adding new file %s' % basepath)
+ _move_file(os.path.join(local_files_dir, basepath),
+ os.path.join(files_dir, basepath))
+ srcuri.append('file://%s' % basepath)
+ update_srcuri = True
+ if update_srcuri:
+ patchfields['SRC_URI'] = ' '.join(srcuri)
+ oe.recipeutils.patch_recipe(rd, recipefile, patchfields)
+ finally:
+ shutil.rmtree(tempdir)
if not 'git://' in orig_src_uri:
logger.info('You will need to update SRC_URI within the recipe to '
'point to a git repository where you have pushed your '
'changes')
- _remove_patch_files(args, removepatches, destpath)
+ _remove_source_files(args, remove_files, destpath)
def _update_recipe_patch(args, config, srctree, rd, config_data):
"""Implement the 'patch' mode of update-recipe"""
@@ -716,83 +835,86 @@ def _update_recipe_patch(args, config, srctree, rd, config_data):
raise DevtoolError('Unable to find initial revision - please specify '
'it with --initial-rev')
- # Find list of existing patches in recipe file
- existing_patches = oe.recipeutils.get_recipe_patches(rd)
+ tempdir = tempfile.mkdtemp(prefix='devtool')
+ try:
+ local_files_dir = tempfile.mkdtemp(dir=tempdir)
+ upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir)
- removepatches = []
- seqpatch_re = re.compile('^([0-9]{4}-)?(.+)')
- if not args.no_remove:
- # Get all patches from source tree and check if any should be removed
- tempdir = tempfile.mkdtemp(prefix='devtool')
- try:
+ remove_files = []
+ if not args.no_remove:
# Get all patches from source tree and check if any should be removed
+ all_patches_dir = tempfile.mkdtemp(dir=tempdir)
upd_p, new_p, del_p = _export_patches(srctree, rd, initial_rev,
- tempdir)
- # Remove deleted patches
- removepatches = del_p.values()
- finally:
- shutil.rmtree(tempdir)
+ all_patches_dir)
+ # Remove deleted local files and patches
+ remove_files = del_f.values() + del_p.values()
- # Get updated patches from source tree
- tempdir = tempfile.mkdtemp(prefix='devtool')
- try:
+ # Get updated patches from source tree
+ patches_dir = tempfile.mkdtemp(dir=tempdir)
upd_p, new_p, del_p = _export_patches(srctree, rd, update_rev,
- tempdir)
-
- # Match up and replace existing patches with corresponding new patches
- updatepatches = False
+ patches_dir)
+ updatefiles = False
updaterecipe = False
destpath = None
+ srcuri = (rd.getVar('SRC_URI', False) or '').split()
if args.append:
- patchfiles = dict((os.path.join(tempdir, key), val) for
- key, val in upd_p.items() + new_p.items())
-
- if patchfiles or removepatches:
+ files = dict((os.path.join(local_files_dir, key), val) for
+ key, val in upd_f.items() + new_f.items())
+ files.update(dict((os.path.join(patches_dir, key), val) for
+ key, val in upd_p.items() + new_p.items()))
+ if files or remove_files:
removevalues = None
- if removepatches:
- srcuri = (rd.getVar('SRC_URI', False) or '').split()
- removedentries, remaining = _remove_patch_entries(
- srcuri, removepatches)
+ if remove_files:
+ removedentries, remaining = _remove_file_entries(
+ srcuri, remove_files)
if removedentries or remaining:
remaining = ['file://' + os.path.basename(item) for
item in remaining]
removevalues = {'SRC_URI': removedentries + remaining}
_, destpath = oe.recipeutils.bbappend_recipe(
- rd, args.append, patchfiles,
+ rd, args.append, files,
removevalues=removevalues)
else:
- logger.info('No patches needed updating')
+ logger.info('No patches or local source files needed updating')
else:
+ # Update existing files
+ for basepath, path in upd_f.iteritems():
+ logger.info('Updating file %s' % basepath)
+ _move_file(os.path.join(local_files_dir, basepath), path)
+ updatefiles = True
for basepath, path in upd_p.iteritems():
logger.info('Updating patch %s' % basepath)
- shutil.move(os.path.join(tempdir, basepath), path)
- updatepatches = True
- srcuri = (rd.getVar('SRC_URI', False) or '').split()
- patchdir = os.path.join(os.path.dirname(recipefile),
- rd.getVar('BPN', True))
- bb.utils.mkdirhier(patchdir)
+ _move_file(os.path.join(patches_dir, basepath), path)
+ updatefiles = True
+ # Add any new files
+ files_dir = os.path.join(os.path.dirname(recipefile),
+ rd.getVar('BPN', True))
+ for basepath, path in new_f.iteritems():
+ logger.info('Adding new file %s' % basepath)
+ _move_file(os.path.join(local_files_dir, basepath),
+ os.path.join(files_dir, basepath))
+ srcuri.append('file://%s' % basepath)
+ updaterecipe = True
for basepath, path in new_p.iteritems():
logger.info('Adding new patch %s' % basepath)
- bb.utils.mkdirhier(patchdir)
- shutil.move(os.path.join(tempdir, basepath),
- os.path.join(patchdir, basepath))
+ _move_file(os.path.join(patches_dir, basepath),
+ os.path.join(files_dir, basepath))
srcuri.append('file://%s' % basepath)
updaterecipe = True
- if removepatches:
- removedentries, _ = _remove_patch_entries(srcuri, removepatches)
- if removedentries:
- updaterecipe = True
+ # Update recipe, if needed
+ if _remove_file_entries(srcuri, remove_files)[0]:
+ updaterecipe = True
if updaterecipe:
logger.info('Updating recipe %s' % os.path.basename(recipefile))
oe.recipeutils.patch_recipe(rd, recipefile,
{'SRC_URI': ' '.join(srcuri)})
- elif not updatepatches:
+ elif not updatefiles:
# Neither patches nor recipe were updated
- logger.info('No patches need updating')
+ logger.info('No patches or files need updating')
finally:
shutil.rmtree(tempdir)
- _remove_patch_files(args, removepatches, destpath)
+ _remove_source_files(args, remove_files, destpath)
def _guess_recipe_update_mode(srctree, rdata):
"""Guess the recipe update mode to use"""
--
2.1.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH v3 10/10] devtool: modify: make bitbake use local files from srctree
2015-09-24 11:52 [PATCH v3 00/10] devtool: improve handling of local source files Markus Lehtonen
` (8 preceding siblings ...)
2015-09-24 11:53 ` [PATCH v3 09/10] devtool: better support for local source files Markus Lehtonen
@ 2015-09-24 11:53 ` Markus Lehtonen
2015-09-28 13:48 ` Paul Eggleton
9 siblings, 1 reply; 18+ messages in thread
From: Markus Lehtonen @ 2015-09-24 11:53 UTC (permalink / raw)
To: openembedded-core; +Cc: Paul Eggleton
This change makes it possible to have local files (non-remote SRC_URI
files, i.e. files that are located in the "recipe space") under the
srctree even if S!=WORKDIR. The files must be placed under the
'local-files' subdirectory.
Complements the previous patch that imports local files into srctree.
[YOCTO #7602]
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
scripts/lib/devtool/standard.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 6b85c8c..78b0d27 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -558,8 +558,13 @@ def modify(args, config, basepath, workspace):
if not os.path.exists(appendpath):
os.makedirs(appendpath)
with open(appendfile, 'w') as f:
- f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n')
- f.write('inherit externalsrc\n')
+ f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n')
+ # Local files can be modified/tracked in separate subdir under srctree
+ # Mostly useful for packages with S != WORKDIR
+ f.write('FILESPATH_prepend := "%s:"\n' %
+ os.path.join(srctree, 'local-files'))
+
+ f.write('\ninherit externalsrc\n')
f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n')
f.write('EXTERNALSRC_pn-%s = "%s"\n' % (args.recipename, srctree))
--
2.1.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* Re: [PATCH v3 10/10] devtool: modify: make bitbake use local files from srctree
2015-09-24 11:53 ` [PATCH v3 10/10] devtool: modify: make bitbake use local files from srctree Markus Lehtonen
@ 2015-09-28 13:48 ` Paul Eggleton
2015-09-30 9:01 ` Markus Lehtonen
0 siblings, 1 reply; 18+ messages in thread
From: Paul Eggleton @ 2015-09-28 13:48 UTC (permalink / raw)
To: Markus Lehtonen; +Cc: openembedded-core
Hi Markus,
On Thursday 24 September 2015 14:53:07 Markus Lehtonen wrote:
> This change makes it possible to have local files (non-remote SRC_URI
> files, i.e. files that are located in the "recipe space") under the
> srctree even if S!=WORKDIR. The files must be placed under the
> 'local-files' subdirectory.
>
> Complements the previous patch that imports local files into srctree.
>
> [YOCTO #7602]
>
> Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
> ---
> scripts/lib/devtool/standard.py | 9 +++++++--
> 1 file changed, 7 insertions(+), 2 deletions(-)
>
> diff --git a/scripts/lib/devtool/standard.py
> b/scripts/lib/devtool/standard.py index 6b85c8c..78b0d27 100644
> --- a/scripts/lib/devtool/standard.py
> +++ b/scripts/lib/devtool/standard.py
> @@ -558,8 +558,13 @@ def modify(args, config, basepath, workspace):
> if not os.path.exists(appendpath):
> os.makedirs(appendpath)
> with open(appendfile, 'w') as f:
> - f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n')
> - f.write('inherit externalsrc\n')
> + f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n')
> + # Local files can be modified/tracked in separate subdir under
> srctree + # Mostly useful for packages with S != WORKDIR
> + f.write('FILESPATH_prepend := "%s:"\n' %
> + os.path.join(srctree, 'local-files'))
Shouldn't this directory be named "oe-local-files"?
Cheers,
Paul
--
Paul Eggleton
Intel Open Source Technology Centre
^ permalink raw reply [flat|nested] 18+ messages in thread* Re: [PATCH v3 10/10] devtool: modify: make bitbake use local files from srctree
2015-09-28 13:48 ` Paul Eggleton
@ 2015-09-30 9:01 ` Markus Lehtonen
2015-09-30 9:21 ` Paul Eggleton
0 siblings, 1 reply; 18+ messages in thread
From: Markus Lehtonen @ 2015-09-30 9:01 UTC (permalink / raw)
To: Paul Eggleton; +Cc: openembedded-core
Hi,
On Mon, 2015-09-28 at 14:48 +0100, Paul Eggleton wrote:
> Hi Markus,
>
> On Thursday 24 September 2015 14:53:07 Markus Lehtonen wrote:
> > This change makes it possible to have local files (non-remote SRC_URI
> > files, i.e. files that are located in the "recipe space") under the
> > srctree even if S!=WORKDIR. The files must be placed under the
> > 'local-files' subdirectory.
> >
> > Complements the previous patch that imports local files into srctree.
> >
> > [YOCTO #7602]
> >
> > Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
> > ---
> > scripts/lib/devtool/standard.py | 9 +++++++--
> > 1 file changed, 7 insertions(+), 2 deletions(-)
> >
> > diff --git a/scripts/lib/devtool/standard.py
> > b/scripts/lib/devtool/standard.py index 6b85c8c..78b0d27 100644
> > --- a/scripts/lib/devtool/standard.py
> > +++ b/scripts/lib/devtool/standard.py
> > @@ -558,8 +558,13 @@ def modify(args, config, basepath, workspace):
> > if not os.path.exists(appendpath):
> > os.makedirs(appendpath)
> > with open(appendfile, 'w') as f:
> > - f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n')
> > - f.write('inherit externalsrc\n')
> > + f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n')
> > + # Local files can be modified/tracked in separate subdir under
> > srctree + # Mostly useful for packages with S != WORKDIR
> > + f.write('FILESPATH_prepend := "%s:"\n' %
> > + os.path.join(srctree, 'local-files'))
>
> Shouldn't this directory be named "oe-local-files"?
Argh, sorry, I had missed this email earlier. Yes, it should be
'oe-local-files'. An updated patch with a fix is available at:
http://cgit.openembedded.org/openembedded-core-contrib/log/?h=marquiz/devtool/localfiles
Thanks,
Markus
^ permalink raw reply [flat|nested] 18+ messages in thread* Re: [PATCH v3 10/10] devtool: modify: make bitbake use local files from srctree
2015-09-30 9:01 ` Markus Lehtonen
@ 2015-09-30 9:21 ` Paul Eggleton
2015-09-30 10:05 ` Markus Lehtonen
0 siblings, 1 reply; 18+ messages in thread
From: Paul Eggleton @ 2015-09-30 9:21 UTC (permalink / raw)
To: Markus Lehtonen; +Cc: openembedded-core
On Wednesday 30 September 2015 12:01:13 Markus Lehtonen wrote:
> Hi,
>
> On Mon, 2015-09-28 at 14:48 +0100, Paul Eggleton wrote:
> > Hi Markus,
> >
> > On Thursday 24 September 2015 14:53:07 Markus Lehtonen wrote:
> > > This change makes it possible to have local files (non-remote SRC_URI
> > > files, i.e. files that are located in the "recipe space") under the
> > > srctree even if S!=WORKDIR. The files must be placed under the
> > > 'local-files' subdirectory.
> > >
> > > Complements the previous patch that imports local files into srctree.
> > >
> > > [YOCTO #7602]
> > >
> > > Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
> > > ---
> > >
> > > scripts/lib/devtool/standard.py | 9 +++++++--
> > > 1 file changed, 7 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/scripts/lib/devtool/standard.py
> > > b/scripts/lib/devtool/standard.py index 6b85c8c..78b0d27 100644
> > > --- a/scripts/lib/devtool/standard.py
> > > +++ b/scripts/lib/devtool/standard.py
> > >
> > > @@ -558,8 +558,13 @@ def modify(args, config, basepath, workspace):
> > > if not os.path.exists(appendpath):
> > > os.makedirs(appendpath)
> > >
> > > with open(appendfile, 'w') as f:
> > > - f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n')
> > > - f.write('inherit externalsrc\n')
> > > + f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n')
> > > + # Local files can be modified/tracked in separate subdir under
> > > srctree + # Mostly useful for packages with S != WORKDIR
> > > + f.write('FILESPATH_prepend := "%s:"\n' %
> > > + os.path.join(srctree, 'local-files'))
> >
> > Shouldn't this directory be named "oe-local-files"?
>
> Argh, sorry, I had missed this email earlier. Yes, it should be
> 'oe-local-files'. An updated patch with a fix is available at:
>
> http://cgit.openembedded.org/openembedded-core-contrib/log/?h=marquiz/devtoo
> l/localfiles
OK, great, but the commit message still has "local-files".
Cheers,
Paul
--
Paul Eggleton
Intel Open Source Technology Centre
^ permalink raw reply [flat|nested] 18+ messages in thread* Re: [PATCH v3 10/10] devtool: modify: make bitbake use local files from srctree
2015-09-30 9:21 ` Paul Eggleton
@ 2015-09-30 10:05 ` Markus Lehtonen
0 siblings, 0 replies; 18+ messages in thread
From: Markus Lehtonen @ 2015-09-30 10:05 UTC (permalink / raw)
To: Paul Eggleton; +Cc: openembedded-core
Hi,
On Wed, 2015-09-30 at 10:21 +0100, Paul Eggleton wrote:
> On Wednesday 30 September 2015 12:01:13 Markus Lehtonen wrote:
> > Hi,
> >
> > On Mon, 2015-09-28 at 14:48 +0100, Paul Eggleton wrote:
> > > Hi Markus,
> > >
> > > On Thursday 24 September 2015 14:53:07 Markus Lehtonen wrote:
> > > > This change makes it possible to have local files (non-remote SRC_URI
> > > > files, i.e. files that are located in the "recipe space") under the
> > > > srctree even if S!=WORKDIR. The files must be placed under the
> > > > 'local-files' subdirectory.
> > > >
> > > > Complements the previous patch that imports local files into srctree.
> > > >
> > > > [YOCTO #7602]
> > > >
> > > > Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
> > > > ---
> > > >
> > > > scripts/lib/devtool/standard.py | 9 +++++++--
> > > > 1 file changed, 7 insertions(+), 2 deletions(-)
> > > >
> > > > diff --git a/scripts/lib/devtool/standard.py
> > > > b/scripts/lib/devtool/standard.py index 6b85c8c..78b0d27 100644
> > > > --- a/scripts/lib/devtool/standard.py
> > > > +++ b/scripts/lib/devtool/standard.py
> > > >
> > > > @@ -558,8 +558,13 @@ def modify(args, config, basepath, workspace):
> > > > if not os.path.exists(appendpath):
> > > > os.makedirs(appendpath)
> > > >
> > > > with open(appendfile, 'w') as f:
> > > > - f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n')
> > > > - f.write('inherit externalsrc\n')
> > > > + f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n')
> > > > + # Local files can be modified/tracked in separate subdir under
> > > > srctree + # Mostly useful for packages with S != WORKDIR
> > > > + f.write('FILESPATH_prepend := "%s:"\n' %
> > > > + os.path.join(srctree, 'local-files'))
> > >
> > > Shouldn't this directory be named "oe-local-files"?
> >
> > Argh, sorry, I had missed this email earlier. Yes, it should be
> > 'oe-local-files'. An updated patch with a fix is available at:
> >
> > http://cgit.openembedded.org/openembedded-core-contrib/log/?h=marquiz/devtoo
> > l/localfiles
>
> OK, great, but the commit message still has "local-files".
Oh my, you're right. Yet another version of the patch in the same
location:
http://cgit.openembedded.org/openembedded-core-contrib/log/?h=marquiz/devtool/localfiles
Thank you for your patience,
Markus
^ permalink raw reply [flat|nested] 18+ messages in thread