* [PATCH] context: Include a command line argument to run all except certain tests
@ 2017-08-03 14:01 leonardo.sandoval.gonzalez
0 siblings, 0 replies; 3+ messages in thread
From: leonardo.sandoval.gonzalez @ 2017-08-03 14:01 UTC (permalink / raw)
To: openembedded-core
From: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
A new command line argument (-R, which is the oposite of current -r) that allows
to run all test cases except the ones indicated through the command line.
Some command line examples:
* Run all except the distro test case:
$ oe-selftest -R distrodata
* Run all except the archiver test case and a single bblayers unit test
$ oe-selftest -R archiver bblayers.BitbakeLayers.test_bitbakelayers_add_remove
[YOCTO #11847]
Signed-off-by: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
---
meta/lib/oeqa/core/context.py | 20 +++++++++++++++++---
meta/lib/oeqa/selftest/context.py | 11 ++++++++---
2 files changed, 25 insertions(+), 6 deletions(-)
diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
index 422e289992..acd547416f 100644
--- a/meta/lib/oeqa/core/context.py
+++ b/meta/lib/oeqa/core/context.py
@@ -41,6 +41,14 @@ class OETestContext(object):
return modules
+ def skipTests(self, skips):
+ if not skips:
+ return
+ for test in self.suites:
+ for skip in skips:
+ if test.id().startswith(skip):
+ setattr(test, 'setUp', lambda: test.skipTest('Skip by the command line argument "%s"' % skip))
+
def loadTests(self, module_paths, modules=[], tests=[],
modules_manifest="", modules_required=[], filters={}):
if modules_manifest:
@@ -50,9 +58,12 @@ class OETestContext(object):
modules_required, filters)
self.suites = self.loader.discover()
- def runTests(self):
+ def runTests(self, skips=[]):
self.runner = self.runnerClass(self, descriptions=False, verbosity=2)
+ # Dinamically skip those tests specified though arguments
+ self.skipTests(skips)
+
self._run_start_time = time.time()
result = self.runner.run(self.suites)
self._run_end_time = time.time()
@@ -128,7 +139,8 @@ class OETestContextExecutor(object):
self.tc_kwargs = {}
self.tc_kwargs['init'] = {}
self.tc_kwargs['load'] = {}
- self.tc_kwargs['run'] = {}
+ self.tc_kwargs['list'] = {}
+ self.tc_kwargs['run'] = {}
self.tc_kwargs['init']['logger'] = self._setup_logger(logger, args)
if args.test_data_file:
@@ -143,6 +155,8 @@ class OETestContextExecutor(object):
else:
self.tc_kwargs['load']['modules'] = []
+ self.tc_kwargs['run']['skips'] = []
+
self.module_paths = args.CASES_PATHS
def _pre_run(self):
@@ -159,7 +173,7 @@ class OETestContextExecutor(object):
sys.exit(1)
if args.list_tests:
- rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['run'])
+ rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['list'])
else:
self._pre_run()
rc = self.tc.runTests(**self.tc_kwargs['run'])
diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py
index 990c761f29..9e90d3c256 100644
--- a/meta/lib/oeqa/selftest/context.py
+++ b/meta/lib/oeqa/selftest/context.py
@@ -25,14 +25,14 @@ class OESelftestTestContext(OETestContext):
self.custommachine = None
self.config_paths = config_paths
- def runTests(self, machine=None):
+ def runTests(self, machine=None, skips=[]):
if machine:
self.custommachine = machine
if machine == 'random':
self.custommachine = choice(self.machines)
self.logger.info('Run tests with custom MACHINE set to: %s' % \
self.custommachine)
- return super(OESelftestTestContext, self).runTests()
+ return super(OESelftestTestContext, self).runTests(skips)
def listTests(self, display_type, machine=None):
return super(OESelftestTestContext, self).listTests(display_type)
@@ -51,6 +51,9 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
group.add_argument('-a', '--run-all-tests', default=False,
action="store_true", dest="run_all_tests",
help='Run all (unhidden) tests')
+ group.add_argument('-R', '--skip-tests', required=False, action='store',
+ nargs='+', dest="skips", default=None,
+ help='Run all (unhidden) tests except the ones specified. Format should be <module>[.<class>[.<test_method>]]')
group.add_argument('-r', '--run-tests', required=False, action='store',
nargs='+', dest="run_tests", default=None,
help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>')
@@ -133,6 +136,8 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
copyfile(self.tc_kwargs['init']['config_paths']['bblayers'],
self.tc_kwargs['init']['config_paths']['bblayers_backup'])
+ self.tc_kwargs['run']['skips'] = args.skips
+
def _pre_run(self):
def _check_required_env_variables(vars):
for var in vars:
@@ -203,7 +208,7 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
sys.exit(1)
if args.list_tests:
- rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['run'])
+ rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['list'])
else:
self._pre_run()
rc = self.tc.runTests(**self.tc_kwargs['run'])
--
2.12.3
^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [PATCH] context: Include a command line argument to run all except certain tests
[not found] <20170802172239.26420-1-leonardo.sandoval.gonzalez@linux.intel.com>
@ 2017-08-07 20:29 ` Aníbal Limón
2017-08-07 20:42 ` Leonardo Sandoval
0 siblings, 1 reply; 3+ messages in thread
From: Aníbal Limón @ 2017-08-07 20:29 UTC (permalink / raw)
To: Leoanardo Sandoval, openembedded-core
Hi,
The code looks ok but i prefer to implement this kind of feature inside
loader instead of at context module, this is because the oeqa loader has
all the logic to get tests and filter it if is necessary.
Cheers,
Anibal
On 08/02/2017 12:22 PM, Leoanardo Sandoval wrote:
> Useful command line argument (-R, which is the oposite of current -r) that allows
> to run all test cases except the ones indicated through the command line.
>
> Some command line examples:
>
> * Run all except the distro test case:
> $ oe-selftest -R distrodata
>
> * Run all except the archiver test case and a single bblayers unit test
> $ oe-selftest -R archiver bblayers.BitbakeLayers.test_bitbakelayers_add_remove
>
> [YOCTO #11847]
>
> Signed-off-by: Leoanardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
> ---
> meta/lib/oeqa/core/context.py | 20 +++++++++++++++++---
> meta/lib/oeqa/selftest/context.py | 11 ++++++++---
> 2 files changed, 25 insertions(+), 6 deletions(-)
>
> diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
> index 422e289992..acd547416f 100644
> --- a/meta/lib/oeqa/core/context.py
> +++ b/meta/lib/oeqa/core/context.py
> @@ -41,6 +41,14 @@ class OETestContext(object):
>
> return modules
>
> + def skipTests(self, skips):
> + if not skips:
> + return
> + for test in self.suites:
> + for skip in skips:
> + if test.id().startswith(skip):
> + setattr(test, 'setUp', lambda: test.skipTest('Skip by the command line argument "%s"' % skip))
> +
> def loadTests(self, module_paths, modules=[], tests=[],
> modules_manifest="", modules_required=[], filters={}):
> if modules_manifest:
> @@ -50,9 +58,12 @@ class OETestContext(object):
> modules_required, filters)
> self.suites = self.loader.discover()
>
> - def runTests(self):
> + def runTests(self, skips=[]):
> self.runner = self.runnerClass(self, descriptions=False, verbosity=2)
>
> + # Dinamically skip those tests specified though arguments
> + self.skipTests(skips)
> +
> self._run_start_time = time.time()
> result = self.runner.run(self.suites)
> self._run_end_time = time.time()
> @@ -128,7 +139,8 @@ class OETestContextExecutor(object):
> self.tc_kwargs = {}
> self.tc_kwargs['init'] = {}
> self.tc_kwargs['load'] = {}
> - self.tc_kwargs['run'] = {}
> + self.tc_kwargs['list'] = {}
> + self.tc_kwargs['run'] = {}
>
> self.tc_kwargs['init']['logger'] = self._setup_logger(logger, args)
> if args.test_data_file:
> @@ -143,6 +155,8 @@ class OETestContextExecutor(object):
> else:
> self.tc_kwargs['load']['modules'] = []
>
> + self.tc_kwargs['run']['skips'] = []
> +
> self.module_paths = args.CASES_PATHS
>
> def _pre_run(self):
> @@ -159,7 +173,7 @@ class OETestContextExecutor(object):
> sys.exit(1)
>
> if args.list_tests:
> - rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['run'])
> + rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['list'])
> else:
> self._pre_run()
> rc = self.tc.runTests(**self.tc_kwargs['run'])
> diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py
> index 990c761f29..9e90d3c256 100644
> --- a/meta/lib/oeqa/selftest/context.py
> +++ b/meta/lib/oeqa/selftest/context.py
> @@ -25,14 +25,14 @@ class OESelftestTestContext(OETestContext):
> self.custommachine = None
> self.config_paths = config_paths
>
> - def runTests(self, machine=None):
> + def runTests(self, machine=None, skips=[]):
> if machine:
> self.custommachine = machine
> if machine == 'random':
> self.custommachine = choice(self.machines)
> self.logger.info('Run tests with custom MACHINE set to: %s' % \
> self.custommachine)
> - return super(OESelftestTestContext, self).runTests()
> + return super(OESelftestTestContext, self).runTests(skips)
>
> def listTests(self, display_type, machine=None):
> return super(OESelftestTestContext, self).listTests(display_type)
> @@ -51,6 +51,9 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
> group.add_argument('-a', '--run-all-tests', default=False,
> action="store_true", dest="run_all_tests",
> help='Run all (unhidden) tests')
> + group.add_argument('-R', '--skip-tests', required=False, action='store',
> + nargs='+', dest="skips", default=None,
> + help='Run all (unhidden) tests except the ones specified. Format should be <module>[.<class>[.<test_method>]]')
> group.add_argument('-r', '--run-tests', required=False, action='store',
> nargs='+', dest="run_tests", default=None,
> help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>')
> @@ -133,6 +136,8 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
> copyfile(self.tc_kwargs['init']['config_paths']['bblayers'],
> self.tc_kwargs['init']['config_paths']['bblayers_backup'])
>
> + self.tc_kwargs['run']['skips'] = args.skips
> +
> def _pre_run(self):
> def _check_required_env_variables(vars):
> for var in vars:
> @@ -203,7 +208,7 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
> sys.exit(1)
>
> if args.list_tests:
> - rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['run'])
> + rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['list'])
> else:
> self._pre_run()
> rc = self.tc.runTests(**self.tc_kwargs['run'])
>
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [PATCH] context: Include a command line argument to run all except certain tests
2017-08-07 20:29 ` [PATCH] context: Include a command line argument to run all except certain tests Aníbal Limón
@ 2017-08-07 20:42 ` Leonardo Sandoval
0 siblings, 0 replies; 3+ messages in thread
From: Leonardo Sandoval @ 2017-08-07 20:42 UTC (permalink / raw)
To: Aníbal Limón; +Cc: openembedded-core
On Mon, 2017-08-07 at 15:29 -0500, Aníbal Limón wrote:
> Hi,
>
> The code looks ok but i prefer to implement this kind of feature inside
> loader instead of at context module, this is because the oeqa loader has
> all the logic to get tests and filter it if is necessary.
I included it at context* because the tests to be skipped come from the
command line and command line stuff is manage there.
>
> Cheers,
> Anibal
>
> On 08/02/2017 12:22 PM, Leoanardo Sandoval wrote:
> > Useful command line argument (-R, which is the oposite of current -r) that allows
> > to run all test cases except the ones indicated through the command line.
> >
> > Some command line examples:
> >
> > * Run all except the distro test case:
> > $ oe-selftest -R distrodata
> >
> > * Run all except the archiver test case and a single bblayers unit test
> > $ oe-selftest -R archiver bblayers.BitbakeLayers.test_bitbakelayers_add_remove
> >
> > [YOCTO #11847]
> >
> > Signed-off-by: Leoanardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
> > ---
> > meta/lib/oeqa/core/context.py | 20 +++++++++++++++++---
> > meta/lib/oeqa/selftest/context.py | 11 ++++++++---
> > 2 files changed, 25 insertions(+), 6 deletions(-)
> >
> > diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
> > index 422e289992..acd547416f 100644
> > --- a/meta/lib/oeqa/core/context.py
> > +++ b/meta/lib/oeqa/core/context.py
> > @@ -41,6 +41,14 @@ class OETestContext(object):
> >
> > return modules
> >
> > + def skipTests(self, skips):
> > + if not skips:
> > + return
> > + for test in self.suites:
> > + for skip in skips:
> > + if test.id().startswith(skip):
> > + setattr(test, 'setUp', lambda: test.skipTest('Skip by the command line argument "%s"' % skip))
> > +
> > def loadTests(self, module_paths, modules=[], tests=[],
> > modules_manifest="", modules_required=[], filters={}):
> > if modules_manifest:
> > @@ -50,9 +58,12 @@ class OETestContext(object):
> > modules_required, filters)
> > self.suites = self.loader.discover()
> >
> > - def runTests(self):
> > + def runTests(self, skips=[]):
> > self.runner = self.runnerClass(self, descriptions=False, verbosity=2)
> >
> > + # Dinamically skip those tests specified though arguments
> > + self.skipTests(skips)
> > +
> > self._run_start_time = time.time()
> > result = self.runner.run(self.suites)
> > self._run_end_time = time.time()
> > @@ -128,7 +139,8 @@ class OETestContextExecutor(object):
> > self.tc_kwargs = {}
> > self.tc_kwargs['init'] = {}
> > self.tc_kwargs['load'] = {}
> > - self.tc_kwargs['run'] = {}
> > + self.tc_kwargs['list'] = {}
> > + self.tc_kwargs['run'] = {}
> >
> > self.tc_kwargs['init']['logger'] = self._setup_logger(logger, args)
> > if args.test_data_file:
> > @@ -143,6 +155,8 @@ class OETestContextExecutor(object):
> > else:
> > self.tc_kwargs['load']['modules'] = []
> >
> > + self.tc_kwargs['run']['skips'] = []
> > +
> > self.module_paths = args.CASES_PATHS
> >
> > def _pre_run(self):
> > @@ -159,7 +173,7 @@ class OETestContextExecutor(object):
> > sys.exit(1)
> >
> > if args.list_tests:
> > - rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['run'])
> > + rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['list'])
> > else:
> > self._pre_run()
> > rc = self.tc.runTests(**self.tc_kwargs['run'])
> > diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py
> > index 990c761f29..9e90d3c256 100644
> > --- a/meta/lib/oeqa/selftest/context.py
> > +++ b/meta/lib/oeqa/selftest/context.py
> > @@ -25,14 +25,14 @@ class OESelftestTestContext(OETestContext):
> > self.custommachine = None
> > self.config_paths = config_paths
> >
> > - def runTests(self, machine=None):
> > + def runTests(self, machine=None, skips=[]):
> > if machine:
> > self.custommachine = machine
> > if machine == 'random':
> > self.custommachine = choice(self.machines)
> > self.logger.info('Run tests with custom MACHINE set to: %s' % \
> > self.custommachine)
> > - return super(OESelftestTestContext, self).runTests()
> > + return super(OESelftestTestContext, self).runTests(skips)
> >
> > def listTests(self, display_type, machine=None):
> > return super(OESelftestTestContext, self).listTests(display_type)
> > @@ -51,6 +51,9 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
> > group.add_argument('-a', '--run-all-tests', default=False,
> > action="store_true", dest="run_all_tests",
> > help='Run all (unhidden) tests')
> > + group.add_argument('-R', '--skip-tests', required=False, action='store',
> > + nargs='+', dest="skips", default=None,
> > + help='Run all (unhidden) tests except the ones specified. Format should be <module>[.<class>[.<test_method>]]')
> > group.add_argument('-r', '--run-tests', required=False, action='store',
> > nargs='+', dest="run_tests", default=None,
> > help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>')
> > @@ -133,6 +136,8 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
> > copyfile(self.tc_kwargs['init']['config_paths']['bblayers'],
> > self.tc_kwargs['init']['config_paths']['bblayers_backup'])
> >
> > + self.tc_kwargs['run']['skips'] = args.skips
> > +
> > def _pre_run(self):
> > def _check_required_env_variables(vars):
> > for var in vars:
> > @@ -203,7 +208,7 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
> > sys.exit(1)
> >
> > if args.list_tests:
> > - rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['run'])
> > + rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['list'])
> > else:
> > self._pre_run()
> > rc = self.tc.runTests(**self.tc_kwargs['run'])
> >
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2017-08-07 20:34 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
[not found] <20170802172239.26420-1-leonardo.sandoval.gonzalez@linux.intel.com>
2017-08-07 20:29 ` [PATCH] context: Include a command line argument to run all except certain tests Aníbal Limón
2017-08-07 20:42 ` Leonardo Sandoval
2017-08-03 14:01 leonardo.sandoval.gonzalez
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox