* [PATCH 1/6] recipetool: avoid second-level subdir when extracting
2015-04-29 13:41 [PATCH 0/6] devtool: enable adding remote source Paul Eggleton
@ 2015-04-29 13:41 ` Paul Eggleton
2015-04-29 13:41 ` [PATCH 2/6] recipetool: ensure git clone is a full clone " Paul Eggleton
` (4 subsequent siblings)
5 siblings, 0 replies; 10+ messages in thread
From: Paul Eggleton @ 2015-04-29 13:41 UTC (permalink / raw)
To: openembedded-core
When -x was specified, we were getting the normal unpack subdirectory
which we don't really want - if there's only one subdirectory unpacked
then we should effectively copy just it to the extraction path, not as a
subdirectory under it.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
scripts/lib/recipetool/create.py | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 1c71b24..1d5bfd9 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -1,6 +1,6 @@
# Recipe creation tool - create command plugin
#
-# Copyright (C) 2014 Intel Corporation
+# Copyright (C) 2014-2015 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -110,11 +110,8 @@ def create_recipe(args):
if '://' in args.source:
# Fetch a URL
srcuri = args.source
- if args.extract_to:
- srctree = args.extract_to
- else:
- tempsrc = tempfile.mkdtemp(prefix='recipetool-')
- srctree = tempsrc
+ tempsrc = tempfile.mkdtemp(prefix='recipetool-')
+ srctree = tempsrc
logger.info('Fetching %s...' % srcuri)
checksums = fetch_source(args.source, srctree)
dirlist = os.listdir(srctree)
@@ -240,6 +237,10 @@ def create_recipe(args):
outlines.append('')
outlines.extend(lines_after)
+ if args.extract_to:
+ shutil.move(srctree, args.extract_to)
+ logger.info('Source extracted to %s' % args.extract_to)
+
if outfile == '-':
sys.stdout.write('\n'.join(outlines) + '\n')
else:
--
2.1.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 2/6] recipetool: ensure git clone is a full clone when extracting
2015-04-29 13:41 [PATCH 0/6] devtool: enable adding remote source Paul Eggleton
2015-04-29 13:41 ` [PATCH 1/6] recipetool: avoid second-level subdir when extracting Paul Eggleton
@ 2015-04-29 13:41 ` Paul Eggleton
2015-04-30 10:17 ` Markus Lehtonen
2015-04-29 13:41 ` [PATCH 3/6] devtool: add: add an option to fetch remote source Paul Eggleton
` (3 subsequent siblings)
5 siblings, 1 reply; 10+ messages in thread
From: Paul Eggleton @ 2015-04-29 13:41 UTC (permalink / raw)
To: openembedded-core
If -x is specified and the specified URI was a git repository, we need
to ensure that the resulting clone is a full clone and not one that has
pointers into the temporary fetch location or DL_DIR. Split out the code
from devtool that already does this for "devtool modify -x" and reuse
that.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
scripts/lib/devtool/standard.py | 8 ++------
scripts/lib/recipetool/create.py | 2 ++
scripts/lib/scriptutils.py | 10 ++++++++++
3 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 94b5e0b..eb41f28 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -23,6 +23,7 @@ import glob
import tempfile
import logging
import argparse
+import scriptutils
from devtool import exec_build_env_command, setup_tinfoil
logger = logging.getLogger('devtool')
@@ -236,12 +237,7 @@ def _extract_source(srctree, keep_temp, devbranch, d):
# Handle if S is set to a subdirectory of the source
srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
- if os.path.exists(os.path.join(srcsubdir, '.git')):
- alternatesfile = os.path.join(srcsubdir, '.git', 'objects', 'info', 'alternates')
- if os.path.exists(alternatesfile):
- # This will have been cloned with -s, so we need to convert it to a full clone
- bb.process.run('git repack -a', cwd=srcsubdir)
- os.remove(alternatesfile)
+ scriptutils.git_convert_full_clone(srcsubdir)
patchdir = os.path.join(srcsubdir, 'patches')
haspatches = False
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 1d5bfd9..15de694 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -22,6 +22,7 @@ import glob
import fnmatch
import re
import logging
+import scriptutils
logger = logging.getLogger('recipetool')
@@ -238,6 +239,7 @@ def create_recipe(args):
outlines.extend(lines_after)
if args.extract_to:
+ scriptutils.git_convert_full_clone(srctree)
shutil.move(srctree, args.extract_to)
logger.info('Source extracted to %s' % args.extract_to)
diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py
index e786126..4af6fd5 100644
--- a/scripts/lib/scriptutils.py
+++ b/scripts/lib/scriptutils.py
@@ -58,3 +58,13 @@ def load_plugins(logger, plugins, pluginpath):
if hasattr(plugin, 'plugin_init'):
plugin.plugin_init(plugins)
plugins.append(plugin)
+
+def git_convert_full_clone(repodir):
+ """If specified directory is a git repository, ensure it's a full clone"""
+ import bb.process
+ if os.path.exists(os.path.join(repodir, '.git')):
+ alternatesfile = os.path.join(repodir, '.git', 'objects', 'info', 'alternates')
+ if os.path.exists(alternatesfile):
+ # This will have been cloned with -s, so we need to convert it to a full clone
+ bb.process.run('git repack -a', cwd=repodir)
+ os.remove(alternatesfile)
--
2.1.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [PATCH 2/6] recipetool: ensure git clone is a full clone when extracting
2015-04-29 13:41 ` [PATCH 2/6] recipetool: ensure git clone is a full clone " Paul Eggleton
@ 2015-04-30 10:17 ` Markus Lehtonen
2015-04-30 10:26 ` Paul Eggleton
0 siblings, 1 reply; 10+ messages in thread
From: Markus Lehtonen @ 2015-04-30 10:17 UTC (permalink / raw)
To: Paul Eggleton; +Cc: openembedded-core
Hi,
On Wed, 2015-04-29 at 14:41 +0100, Paul Eggleton wrote:
> If -x is specified and the specified URI was a git repository, we need
> to ensure that the resulting clone is a full clone and not one that has
> pointers into the temporary fetch location or DL_DIR. Split out the code
> from devtool that already does this for "devtool modify -x" and reuse
> that.
>
> Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
> ---
> scripts/lib/devtool/standard.py | 8 ++------
> scripts/lib/recipetool/create.py | 2 ++
> scripts/lib/scriptutils.py | 10 ++++++++++
> 3 files changed, 14 insertions(+), 6 deletions(-)
>
> diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
> index 94b5e0b..eb41f28 100644
> --- a/scripts/lib/devtool/standard.py
> +++ b/scripts/lib/devtool/standard.py
> @@ -23,6 +23,7 @@ import glob
> import tempfile
> import logging
> import argparse
> +import scriptutils
> from devtool import exec_build_env_command, setup_tinfoil
>
> logger = logging.getLogger('devtool')
> @@ -236,12 +237,7 @@ def _extract_source(srctree, keep_temp, devbranch, d):
> # Handle if S is set to a subdirectory of the source
> srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
>
> - if os.path.exists(os.path.join(srcsubdir, '.git')):
> - alternatesfile = os.path.join(srcsubdir, '.git', 'objects', 'info', 'alternates')
> - if os.path.exists(alternatesfile):
> - # This will have been cloned with -s, so we need to convert it to a full clone
> - bb.process.run('git repack -a', cwd=srcsubdir)
> - os.remove(alternatesfile)
> + scriptutils.git_convert_full_clone(srcsubdir)
>
> patchdir = os.path.join(srcsubdir, 'patches')
> haspatches = False
> diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
> index 1d5bfd9..15de694 100644
> --- a/scripts/lib/recipetool/create.py
> +++ b/scripts/lib/recipetool/create.py
> @@ -22,6 +22,7 @@ import glob
> import fnmatch
> import re
> import logging
> +import scriptutils
>
> logger = logging.getLogger('recipetool')
>
> @@ -238,6 +239,7 @@ def create_recipe(args):
> outlines.extend(lines_after)
>
> if args.extract_to:
> + scriptutils.git_convert_full_clone(srctree)
> shutil.move(srctree, args.extract_to)
> logger.info('Source extracted to %s' % args.extract_to)
>
> diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py
> index e786126..4af6fd5 100644
> --- a/scripts/lib/scriptutils.py
> +++ b/scripts/lib/scriptutils.py
> @@ -58,3 +58,13 @@ def load_plugins(logger, plugins, pluginpath):
> if hasattr(plugin, 'plugin_init'):
> plugin.plugin_init(plugins)
> plugins.append(plugin)
> +
> +def git_convert_full_clone(repodir):
> + """If specified directory is a git repository, ensure it's a full clone"""
> + import bb.process
> + if os.path.exists(os.path.join(repodir, '.git')):
> + alternatesfile = os.path.join(repodir, '.git', 'objects', 'info', 'alternates')
> + if os.path.exists(alternatesfile):
> + # This will have been cloned with -s, so we need to convert it to a full clone
> + bb.process.run('git repack -a', cwd=repodir)
> + os.remove(alternatesfile)
> --
> 2.1.0
>
I just find the term "full clone" a bit misleading here. I somehow tend
to associate it with a mirrored clone / cloning all refs / cloning with
full history. Could the term used above be "independent clone" or
"standalone clone" or something?
Thanks,
Markus Lehtonen
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [PATCH 2/6] recipetool: ensure git clone is a full clone when extracting
2015-04-30 10:17 ` Markus Lehtonen
@ 2015-04-30 10:26 ` Paul Eggleton
2015-05-01 15:59 ` Paul Eggleton
0 siblings, 1 reply; 10+ messages in thread
From: Paul Eggleton @ 2015-04-30 10:26 UTC (permalink / raw)
To: Markus Lehtonen; +Cc: openembedded-core
Hi Markus,
On Thursday 30 April 2015 13:17:41 Markus Lehtonen wrote:
> On Wed, 2015-04-29 at 14:41 +0100, Paul Eggleton wrote:
> > If -x is specified and the specified URI was a git repository, we need
> > to ensure that the resulting clone is a full clone and not one that has
> > pointers into the temporary fetch location or DL_DIR. Split out the code
> > from devtool that already does this for "devtool modify -x" and reuse
> > that.
> >
> > Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
> > ---
> >
> > scripts/lib/devtool/standard.py | 8 ++------
> > scripts/lib/recipetool/create.py | 2 ++
> > scripts/lib/scriptutils.py | 10 ++++++++++
> > 3 files changed, 14 insertions(+), 6 deletions(-)
> >
> > diff --git a/scripts/lib/devtool/standard.py
> > b/scripts/lib/devtool/standard.py index 94b5e0b..eb41f28 100644
> > --- a/scripts/lib/devtool/standard.py
> > +++ b/scripts/lib/devtool/standard.py
> > @@ -23,6 +23,7 @@ import glob
> >
> > import tempfile
> > import logging
> > import argparse
> >
> > +import scriptutils
> >
> > from devtool import exec_build_env_command, setup_tinfoil
> >
> > logger = logging.getLogger('devtool')
> >
> > @@ -236,12 +237,7 @@ def _extract_source(srctree, keep_temp, devbranch,
d):
> > # Handle if S is set to a subdirectory of the source
> > srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir,
> > workdir).split(os.sep)[0])>
> > - if os.path.exists(os.path.join(srcsubdir, '.git')):
> > - alternatesfile = os.path.join(srcsubdir, '.git', 'objects',
> > 'info', 'alternates') - if os.path.exists(alternatesfile):
> > - # This will have been cloned with -s, so we need to
> > convert it to a full clone - bb.process.run('git repack
> > -a', cwd=srcsubdir)
> > - os.remove(alternatesfile)
> > + scriptutils.git_convert_full_clone(srcsubdir)
> >
> > patchdir = os.path.join(srcsubdir, 'patches')
> > haspatches = False
> >
> > diff --git a/scripts/lib/recipetool/create.py
> > b/scripts/lib/recipetool/create.py index 1d5bfd9..15de694 100644
> > --- a/scripts/lib/recipetool/create.py
> > +++ b/scripts/lib/recipetool/create.py
> > @@ -22,6 +22,7 @@ import glob
> >
> > import fnmatch
> > import re
> > import logging
> >
> > +import scriptutils
> >
> > logger = logging.getLogger('recipetool')
> >
> > @@ -238,6 +239,7 @@ def create_recipe(args):
> > outlines.extend(lines_after)
> >
> > if args.extract_to:
> > + scriptutils.git_convert_full_clone(srctree)
> >
> > shutil.move(srctree, args.extract_to)
> > logger.info('Source extracted to %s' % args.extract_to)
> >
> > diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py
> > index e786126..4af6fd5 100644
> > --- a/scripts/lib/scriptutils.py
> > +++ b/scripts/lib/scriptutils.py
> >
> > @@ -58,3 +58,13 @@ def load_plugins(logger, plugins, pluginpath):
> > if hasattr(plugin, 'plugin_init'):
> > plugin.plugin_init(plugins)
> > plugins.append(plugin)
> >
> > +
> > +def git_convert_full_clone(repodir):
> > + """If specified directory is a git repository, ensure it's a full
> > clone""" + import bb.process
> > + if os.path.exists(os.path.join(repodir, '.git')):
> > + alternatesfile = os.path.join(repodir, '.git', 'objects', 'info',
> > 'alternates') + if os.path.exists(alternatesfile):
> > + # This will have been cloned with -s, so we need to convert
> > it to a full clone + bb.process.run('git repack -a',
> > cwd=repodir)
> > + os.remove(alternatesfile)
>
> I just find the term "full clone" a bit misleading here. I somehow tend
> to associate it with a mirrored clone / cloning all refs / cloning with
> full history. Could the term used above be "independent clone" or
> "standalone clone" or something?
Yep, sounds reasonable - I'll rename it in a v2.
Cheers,
Paul
--
Paul Eggleton
Intel Open Source Technology Centre
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [PATCH 2/6] recipetool: ensure git clone is a full clone when extracting
2015-04-30 10:26 ` Paul Eggleton
@ 2015-05-01 15:59 ` Paul Eggleton
0 siblings, 0 replies; 10+ messages in thread
From: Paul Eggleton @ 2015-05-01 15:59 UTC (permalink / raw)
To: openembedded-core
On Thursday 30 April 2015 11:26:34 Paul Eggleton wrote:
> > I just find the term "full clone" a bit misleading here. I somehow tend
> > to associate it with a mirrored clone / cloning all refs / cloning with
> > full history. Could the term used above be "independent clone" or
> > "standalone clone" or something?
>
> Yep, sounds reasonable - I'll rename it in a v2.
Actually I'm thinking it's not worth sending a v2, I've just updated it on the
branch since it's a straight rename.
Cheers,
Paul
--
Paul Eggleton
Intel Open Source Technology Centre
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH 3/6] devtool: add: add an option to fetch remote source
2015-04-29 13:41 [PATCH 0/6] devtool: enable adding remote source Paul Eggleton
2015-04-29 13:41 ` [PATCH 1/6] recipetool: avoid second-level subdir when extracting Paul Eggleton
2015-04-29 13:41 ` [PATCH 2/6] recipetool: ensure git clone is a full clone " Paul Eggleton
@ 2015-04-29 13:41 ` Paul Eggleton
2015-04-29 13:41 ` [PATCH 4/6] devtool: add: use the appropriate file naming and versioning for SCM recipes Paul Eggleton
` (2 subsequent siblings)
5 siblings, 0 replies; 10+ messages in thread
From: Paul Eggleton @ 2015-04-29 13:41 UTC (permalink / raw)
To: openembedded-core
Add a -f/--fetch option to fetch a remote URI (into the already
specified source tree path) and set this as SRC_URI within the recipe.
This simply wraps around the existing functionality in recipetool.
Implements [YOCTO #7644].
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
scripts/lib/devtool/standard.py | 25 ++++++++++++++++++++++---
1 file changed, 22 insertions(+), 3 deletions(-)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index eb41f28..482c418 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -46,6 +46,19 @@ def add(args, config, basepath, workspace):
return -1
srctree = os.path.abspath(args.srctree)
+ if os.path.exists(srctree):
+ if args.fetch:
+ if not os.path.isdir(srctree):
+ logger.error("Cannot fetch into source tree path %s as it exists and is not a directory" % srctree)
+ return 1
+ elif os.listdir(srctree):
+ logger.error("Cannot fetch into source tree path %s as it already exists and is non-empty" % srctree)
+ return 1
+ else:
+ if not args.fetch:
+ logger.error("Specified source tree %s could not be found" % srctree)
+ return 1
+
appendpath = os.path.join(config.workspace_path, 'appends')
if not os.path.exists(appendpath):
os.makedirs(appendpath)
@@ -64,7 +77,13 @@ def add(args, config, basepath, workspace):
color = 'always'
else:
color = args.color
- stdout, stderr = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s %s' % (color, recipefile, srctree))
+ extracmdopts = ''
+ if args.fetch:
+ source = args.fetch
+ extracmdopts = '-x %s' % srctree
+ else:
+ source = srctree
+ stdout, stderr = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, recipefile, source, extracmdopts))
logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
_add_md5(config, args.recipename, recipefile)
@@ -670,11 +689,11 @@ def build(args, config, basepath, workspace):
def register_commands(subparsers, context):
parser_add = subparsers.add_parser('add', help='Add a new recipe',
- description='Adds a new recipe',
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ description='Adds a new recipe')
parser_add.add_argument('recipename', help='Name for new recipe to add')
parser_add.add_argument('srctree', help='Path to external source tree')
parser_add.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
+ parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree', metavar='URI')
parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
parser_add.set_defaults(func=add)
--
2.1.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 4/6] devtool: add: use the appropriate file naming and versioning for SCM recipes
2015-04-29 13:41 [PATCH 0/6] devtool: enable adding remote source Paul Eggleton
` (2 preceding siblings ...)
2015-04-29 13:41 ` [PATCH 3/6] devtool: add: add an option to fetch remote source Paul Eggleton
@ 2015-04-29 13:41 ` Paul Eggleton
2015-04-29 13:41 ` [PATCH 5/6] recipetool: allow specifying SRCREV via rev= in URI Paul Eggleton
2015-04-29 13:42 ` [PATCH 6/6] oe-selftest: devtool: add some tests for devtool add -f Paul Eggleton
5 siblings, 0 replies; 10+ messages in thread
From: Paul Eggleton @ 2015-04-29 13:41 UTC (permalink / raw)
To: openembedded-core
* Recipes that fetch from git, svn or hg by OpenEmbedded convention
should normally be named with this as a suffix, since PV is meant to
be set appropriately within the recipe, so follow this. In order to
make this work we need to be able to have the version independent from
the file name, so add a -V option to recipetool create to allow this
to be specified.
* If -V is specified on the devtool add command line, ensure at PV gets
set to include this version.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
scripts/lib/devtool/standard.py | 14 +++++++++++++-
scripts/lib/recipetool/create.py | 15 ++++++++++++---
2 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 482c418..16f737e 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -65,11 +65,21 @@ def add(args, config, basepath, workspace):
recipedir = os.path.join(config.workspace_path, 'recipes', args.recipename)
bb.utils.mkdirhier(recipedir)
+ rfv = None
if args.version:
if '_' in args.version or ' ' in args.version:
logger.error('Invalid version string "%s"' % args.version)
return -1
- bp = "%s_%s" % (args.recipename, args.version)
+ rfv = args.version
+ if args.fetch:
+ if args.fetch.startswith('git://'):
+ rfv = 'git'
+ elif args.fetch.startswith('svn://'):
+ rfv = 'svn'
+ elif args.fetch.startswith('hg://'):
+ rfv = 'hg'
+ if rfv:
+ bp = "%s_%s" % (args.recipename, rfv)
else:
bp = args.recipename
recipefile = os.path.join(recipedir, "%s.bb" % bp)
@@ -83,6 +93,8 @@ def add(args, config, basepath, workspace):
extracmdopts = '-x %s' % srctree
else:
source = srctree
+ if args.version:
+ extracmdopts += ' -V %s' % args.version
stdout, stderr = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, recipefile, source, extracmdopts))
logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 15de694..96062ab 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -187,9 +187,17 @@ def create_recipe(args):
pn = recipefn
pv = None
+ if args.version:
+ pv = args.version
+
+ if pv and pv not in 'git svn hg'.split():
+ realpv = pv
+ else:
+ realpv = None
+
if srcuri:
- if pv and pv not in 'git svn hg'.split():
- srcuri = srcuri.replace(pv, '${PV}')
+ if realpv:
+ srcuri = srcuri.replace(realpv, '${PV}')
else:
lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
lines_before.append('SRC_URI = "%s"' % srcuri)
@@ -201,7 +209,7 @@ def create_recipe(args):
if srcuri and supports_srcrev(srcuri):
lines_before.append('')
lines_before.append('# Modify these as desired')
- lines_before.append('PV = "1.0+git${SRCPV}"')
+ lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0'))
lines_before.append('SRCREV = "${AUTOREV}"')
lines_before.append('')
@@ -418,5 +426,6 @@ def register_command(subparsers):
parser_create.add_argument('-o', '--outfile', help='Specify filename for recipe to create', required=True)
parser_create.add_argument('-m', '--machine', help='Make recipe machine-specific as opposed to architecture-specific', action='store_true')
parser_create.add_argument('-x', '--extract-to', metavar='EXTRACTPATH', help='Assuming source is a URL, fetch it and extract it to the directory specified as %(metavar)s')
+ parser_create.add_argument('-V', '--version', help='Version to use within recipe (PV)')
parser_create.set_defaults(func=create_recipe)
--
2.1.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 5/6] recipetool: allow specifying SRCREV via rev= in URI
2015-04-29 13:41 [PATCH 0/6] devtool: enable adding remote source Paul Eggleton
` (3 preceding siblings ...)
2015-04-29 13:41 ` [PATCH 4/6] devtool: add: use the appropriate file naming and versioning for SCM recipes Paul Eggleton
@ 2015-04-29 13:41 ` Paul Eggleton
2015-04-29 13:42 ` [PATCH 6/6] oe-selftest: devtool: add some tests for devtool add -f Paul Eggleton
5 siblings, 0 replies; 10+ messages in thread
From: Paul Eggleton @ 2015-04-29 13:41 UTC (permalink / raw)
To: openembedded-core
Provide a means to set SRCREV immediately by using rev= in the URI; if
it is specified then it is removed before setting SRC_URI and SRCREV is
set to the same value instead.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
scripts/lib/recipetool/create.py | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 96062ab..36bd6d9 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -58,13 +58,13 @@ class RecipeHandler():
-def fetch_source(uri, destdir):
+def fetch_source(uri, destdir, srcrev):
import bb.data
bb.utils.mkdirhier(destdir)
localdata = bb.data.createCopy(tinfoil.config_data)
bb.data.update_data(localdata)
localdata.setVar('BB_STRICT_CHECKSUM', '')
- localdata.setVar('SRCREV', '${AUTOREV}')
+ localdata.setVar('SRCREV', srcrev)
ret = (None, None)
olddir = os.getcwd()
try:
@@ -88,6 +88,9 @@ def fetch_source(uri, destdir):
def supports_srcrev(uri):
localdata = bb.data.createCopy(tinfoil.config_data)
+ # This is a bit sad, but if you don't have this set there can be some
+ # odd interactions with the urldata cache which lead to errors
+ localdata.setVar('SRCREV', '${AUTOREV}')
bb.data.update_data(localdata)
fetcher = bb.fetch2.Fetch([uri], localdata)
urldata = fetcher.ud
@@ -108,13 +111,19 @@ def create_recipe(args):
checksums = (None, None)
tempsrc = ''
srcsubdir = ''
+ srcrev = '${AUTOREV}'
if '://' in args.source:
# Fetch a URL
srcuri = args.source
+ rev_re = re.compile(';rev=([^;]+)')
+ res = rev_re.search(srcuri)
+ if res:
+ srcrev = res.group(1)
+ srcuri = rev_re.sub('', srcuri)
tempsrc = tempfile.mkdtemp(prefix='recipetool-')
srctree = tempsrc
logger.info('Fetching %s...' % srcuri)
- checksums = fetch_source(args.source, srctree)
+ checksums = fetch_source(args.source, srctree, srcrev)
dirlist = os.listdir(srctree)
if 'git.indirectionsymlink' in dirlist:
dirlist.remove('git.indirectionsymlink')
@@ -210,7 +219,7 @@ def create_recipe(args):
lines_before.append('')
lines_before.append('# Modify these as desired')
lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0'))
- lines_before.append('SRCREV = "${AUTOREV}"')
+ lines_before.append('SRCREV = "%s"' % srcrev)
lines_before.append('')
if srcsubdir and pv:
--
2.1.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 6/6] oe-selftest: devtool: add some tests for devtool add -f
2015-04-29 13:41 [PATCH 0/6] devtool: enable adding remote source Paul Eggleton
` (4 preceding siblings ...)
2015-04-29 13:41 ` [PATCH 5/6] recipetool: allow specifying SRCREV via rev= in URI Paul Eggleton
@ 2015-04-29 13:42 ` Paul Eggleton
5 siblings, 0 replies; 10+ messages in thread
From: Paul Eggleton @ 2015-04-29 13:42 UTC (permalink / raw)
To: openembedded-core
Tests the following aspects of "devtool add" with the -f option:
* Works for a git and https URI
* Extracts to the appropriate directory
* Uses the correct recipe file name with and without -V
* Sets SRC_URI and S appropriately with and without -V
* Sets SRCREV if rev= is specified in the URI
A lot of this functionality relies on "recipetool create" which
"devtool add" wraps around, so the associated behaviour of that is also
being tested here.
Refactor out the code to check a recipe for variable values / inherits
at the same time so we can use it to check the generated recipe.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
meta/lib/oeqa/selftest/devtool.py | 148 ++++++++++++++++++++++++++++++--------
1 file changed, 118 insertions(+), 30 deletions(-)
diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
index 6881835..1a506d9 100644
--- a/meta/lib/oeqa/selftest/devtool.py
+++ b/meta/lib/oeqa/selftest/devtool.py
@@ -35,6 +35,24 @@ class DevtoolTests(oeSelfTest):
self.assertNotIn(tempdir, result.output)
self.assertIn(workspacedir, result.output)
+ def _test_recipe_contents(self, recipefile, checkvars, checkinherits):
+ with open(recipefile, 'r') as f:
+ for line in f:
+ if '=' in line:
+ splitline = line.split('=', 1)
+ var = splitline[0].rstrip()
+ value = splitline[1].strip().strip('"')
+ if var in checkvars:
+ needvalue = checkvars.pop(var)
+ self.assertEqual(value, needvalue, 'values for %s do not match' % var)
+ if line.startswith('inherit '):
+ inherits = line.split()[1:]
+
+ self.assertEqual(checkvars, {}, 'Some variables not found: %s' % checkvars)
+
+ for inherit in checkinherits:
+ self.assertIn(inherit, inherits, 'Missing inherit of %s' % inherit)
+
def test_recipetool_create(self):
# Try adding a recipe
tempdir = tempfile.mkdtemp(prefix='devtoolqa')
@@ -51,19 +69,7 @@ class DevtoolTests(oeSelfTest):
checkvars['SRC_URI'] = 'https://fedorahosted.org/releases/l/o/logrotate/logrotate-${PV}.tar.gz'
checkvars['SRC_URI[md5sum]'] = '99e08503ef24c3e2e3ff74cc5f3be213'
checkvars['SRC_URI[sha256sum]'] = 'f6ba691f40e30e640efa2752c1f9499a3f9738257660994de70a45fe00d12b64'
- with open(recipefile, 'r') as f:
- for line in f:
- if '=' in line:
- splitline = line.split('=', 1)
- var = splitline[0].rstrip()
- value = splitline[1].strip().strip('"')
- if var in checkvars:
- needvalue = checkvars.pop(var)
- self.assertEqual(value, needvalue)
- if line.startswith('inherit '):
- inherits = line.split()[1:]
-
- self.assertEqual(checkvars, {}, 'Some variables not found')
+ self._test_recipe_contents(recipefile, checkvars, [])
def test_recipetool_create_git(self):
# Ensure we have the right data in shlibs/pkgdata
@@ -84,23 +90,8 @@ class DevtoolTests(oeSelfTest):
checkvars['PV'] = '1.0+git${SRCPV}'
checkvars['SRC_URI'] = srcuri
checkvars['DEPENDS'] = 'libpng pango libx11 libxext jpeg'
- inherits = []
- with open(recipefile, 'r') as f:
- for line in f:
- if '=' in line:
- splitline = line.split('=', 1)
- var = splitline[0].rstrip()
- value = splitline[1].strip().strip('"')
- if var in checkvars:
- needvalue = checkvars.pop(var)
- self.assertEqual(value, needvalue)
- if line.startswith('inherit '):
- inherits = line.split()[1:]
-
- self.assertEqual(checkvars, {}, 'Some variables not found')
-
- self.assertIn('autotools', inherits, 'Missing inherit of autotools')
- self.assertIn('pkgconfig', inherits, 'Missing inherit of pkgconfig')
+ inherits = ['autotools', 'pkgconfig']
+ self._test_recipe_contents(recipefile, checkvars, inherits)
def test_devtool_add(self):
# Check preconditions
@@ -176,6 +167,103 @@ class DevtoolTests(oeSelfTest):
self.assertFalse(matches, 'Stamp files exist for recipe libftdi that should have been cleaned')
self.assertFalse(os.path.isfile(os.path.join(staging_libdir, 'libftdi1.so.2.1.0')), 'libftdi binary still found in STAGING_LIBDIR after cleaning')
+ def test_devtool_add_fetch(self):
+ # Check preconditions
+ workspacedir = os.path.join(self.builddir, 'workspace')
+ self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ # Fetch source
+ tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+ self.track_for_cleanup(tempdir)
+ testver = '0.23'
+ url = 'https://pypi.python.org/packages/source/J/MarkupSafe/MarkupSafe-%s.tar.gz' % testver
+ testrecipe = 'python-markupsafe'
+ srcdir = os.path.join(tempdir, testrecipe)
+ # Test devtool add
+ self.track_for_cleanup(workspacedir)
+ self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe)
+ self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+ result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url))
+ self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')
+ self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory')
+ # Test devtool status
+ result = runCmd('devtool status')
+ self.assertIn(testrecipe, result.output)
+ self.assertIn(srcdir, result.output)
+ # Check recipe
+ recipefile = get_bb_var('FILE', testrecipe)
+ self.assertIn('%s.bb' % testrecipe, recipefile, 'Recipe file incorrectly named')
+ checkvars = {}
+ checkvars['S'] = '${WORKDIR}/MarkupSafe-%s' % testver
+ checkvars['SRC_URI'] = url
+ self._test_recipe_contents(recipefile, checkvars, [])
+ # Try with version specified
+ result = runCmd('devtool reset -n %s' % testrecipe)
+ shutil.rmtree(srcdir)
+ result = runCmd('devtool add %s %s -f %s -V %s' % (testrecipe, srcdir, url, testver))
+ self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory')
+ # Test devtool status
+ result = runCmd('devtool status')
+ self.assertIn(testrecipe, result.output)
+ self.assertIn(srcdir, result.output)
+ # Check recipe
+ recipefile = get_bb_var('FILE', testrecipe)
+ self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named')
+ checkvars = {}
+ checkvars['S'] = '${WORKDIR}/MarkupSafe-${PV}'
+ checkvars['SRC_URI'] = url.replace(testver, '${PV}')
+ self._test_recipe_contents(recipefile, checkvars, [])
+
+ def test_devtool_add_fetch_git(self):
+ # Check preconditions
+ workspacedir = os.path.join(self.builddir, 'workspace')
+ self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+ # Fetch source
+ tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+ self.track_for_cleanup(tempdir)
+ url = 'git://git.yoctoproject.org/libmatchbox'
+ checkrev = '462f0652055d89c648ddd54fd7b03f175c2c6973'
+ testrecipe = 'libmatchbox2'
+ srcdir = os.path.join(tempdir, testrecipe)
+ # Test devtool add
+ self.track_for_cleanup(workspacedir)
+ self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe)
+ self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+ result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url))
+ self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')
+ self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure.ac in source directory')
+ # Test devtool status
+ result = runCmd('devtool status')
+ self.assertIn(testrecipe, result.output)
+ self.assertIn(srcdir, result.output)
+ # Check recipe
+ recipefile = get_bb_var('FILE', testrecipe)
+ self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named')
+ checkvars = {}
+ checkvars['S'] = '${WORKDIR}/git'
+ checkvars['PV'] = '1.0+git${SRCPV}'
+ checkvars['SRC_URI'] = url
+ checkvars['SRCREV'] = '${AUTOREV}'
+ self._test_recipe_contents(recipefile, checkvars, [])
+ # Try with revision and version specified
+ result = runCmd('devtool reset -n %s' % testrecipe)
+ shutil.rmtree(srcdir)
+ url_rev = '%s;rev=%s' % (url, checkrev)
+ result = runCmd('devtool add %s %s -f "%s" -V 1.5' % (testrecipe, srcdir, url_rev))
+ self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure.ac in source directory')
+ # Test devtool status
+ result = runCmd('devtool status')
+ self.assertIn(testrecipe, result.output)
+ self.assertIn(srcdir, result.output)
+ # Check recipe
+ recipefile = get_bb_var('FILE', testrecipe)
+ self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named')
+ checkvars = {}
+ checkvars['S'] = '${WORKDIR}/git'
+ checkvars['PV'] = '1.5+git${SRCPV}'
+ checkvars['SRC_URI'] = url
+ checkvars['SRCREV'] = checkrev
+ self._test_recipe_contents(recipefile, checkvars, [])
+
def test_devtool_modify(self):
# Check preconditions
workspacedir = os.path.join(self.builddir, 'workspace')
--
2.1.0
^ permalink raw reply related [flat|nested] 10+ messages in thread