* [PATCH 18/32] oeqa: Move common files to oeqa/files instead of runtime only
From: Aníbal Limón @ 2016-12-06 21:44 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
Those files are used by runtime and sdk test cases, so move to
base directory of oeqa module.
[YOCTO #10599]
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
meta/lib/oeqa/{runtime => }/files/test.c | 0
meta/lib/oeqa/{runtime => }/files/test.cpp | 0
meta/lib/oeqa/{runtime => }/files/test.pl | 0
meta/lib/oeqa/{runtime => }/files/test.py | 0
4 files changed, 0 insertions(+), 0 deletions(-)
rename meta/lib/oeqa/{runtime => }/files/test.c (100%)
rename meta/lib/oeqa/{runtime => }/files/test.cpp (100%)
rename meta/lib/oeqa/{runtime => }/files/test.pl (100%)
rename meta/lib/oeqa/{runtime => }/files/test.py (100%)
diff --git a/meta/lib/oeqa/runtime/files/test.c b/meta/lib/oeqa/files/test.c
similarity index 100%
rename from meta/lib/oeqa/runtime/files/test.c
rename to meta/lib/oeqa/files/test.c
diff --git a/meta/lib/oeqa/runtime/files/test.cpp b/meta/lib/oeqa/files/test.cpp
similarity index 100%
rename from meta/lib/oeqa/runtime/files/test.cpp
rename to meta/lib/oeqa/files/test.cpp
diff --git a/meta/lib/oeqa/runtime/files/test.pl b/meta/lib/oeqa/files/test.pl
similarity index 100%
rename from meta/lib/oeqa/runtime/files/test.pl
rename to meta/lib/oeqa/files/test.pl
diff --git a/meta/lib/oeqa/runtime/files/test.py b/meta/lib/oeqa/files/test.py
similarity index 100%
rename from meta/lib/oeqa/runtime/files/test.py
rename to meta/lib/oeqa/files/test.py
--
2.1.4
^ permalink raw reply
* [PATCH 16/32] oeqa/core: Change name of d to td
From: Aníbal Limón @ 2016-12-06 21:44 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
The d variable references the test data into a test context, so
makes a more sense to call it: td (test data).
[YOCTO #10231]
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
meta/lib/oeqa/core/case.py | 22 ++++++++++----------
meta/lib/oeqa/core/cases/example/test_basic.py | 8 ++++----
meta/lib/oeqa/core/context.py | 28 +++++++++++++-------------
meta/lib/oeqa/core/decorator/data.py | 10 ++++-----
meta/lib/oeqa/core/loader.py | 2 +-
meta/lib/oeqa/core/tests/cases/data.py | 6 +++---
6 files changed, 38 insertions(+), 38 deletions(-)
diff --git a/meta/lib/oeqa/core/case.py b/meta/lib/oeqa/core/case.py
index a47cb19..d2dbf20 100644
--- a/meta/lib/oeqa/core/case.py
+++ b/meta/lib/oeqa/core/case.py
@@ -5,30 +5,30 @@ import unittest
from oeqa.core.exception import OEQAMissingVariable
-def _validate_data_vars(d, data_vars, type_msg):
- if data_vars:
- for v in data_vars:
- if not v in d:
+def _validate_td_vars(td, td_vars, type_msg):
+ if td_vars:
+ for v in td_vars:
+ if not v in td:
raise OEQAMissingVariable("Test %s need %s variable but"\
- " isn't into d" % (type_msg, v))
+ " isn't into td" % (type_msg, v))
class OETestCase(unittest.TestCase):
# TestContext and Logger instance set by OETestLoader.
tc = None
logger = None
- # d has all the variables needed by the test cases
+ # td has all the variables needed by the test cases
# is the same across all the test cases.
- d = None
+ td = None
- # data_vars has the variables needed by a test class
- # or test case instance, if some var isn't into d a
+ # td_vars has the variables needed by a test class
+ # or test case instance, if some var isn't into td a
# OEMissingVariable exception is raised
- data_vars = None
+ td_vars = None
@classmethod
def _oeSetUpClass(clss):
- _validate_data_vars(clss.d, clss.data_vars, "class")
+ _validate_td_vars(clss.td, clss.td_vars, "class")
clss.setUpClassMethod()
@classmethod
diff --git a/meta/lib/oeqa/core/cases/example/test_basic.py b/meta/lib/oeqa/core/cases/example/test_basic.py
index 8b404fe..11cf380 100644
--- a/meta/lib/oeqa/core/cases/example/test_basic.py
+++ b/meta/lib/oeqa/core/cases/example/test_basic.py
@@ -6,10 +6,10 @@ from oeqa.core.decorator.depends import OETestDepends
class OETestExample(OETestCase):
def test_example(self):
- self.logger.info('IMAGE: %s' % self.d.get('IMAGE'))
- self.assertEqual('core-image-minimal', self.d.get('IMAGE'))
- self.logger.info('ARCH: %s' % self.d.get('ARCH'))
- self.assertEqual('x86', self.d.get('ARCH'))
+ self.logger.info('IMAGE: %s' % self.td.get('IMAGE'))
+ self.assertEqual('core-image-minimal', self.td.get('IMAGE'))
+ self.logger.info('ARCH: %s' % self.td.get('ARCH'))
+ self.assertEqual('x86', self.td.get('ARCH'))
class OETestExampleDepend(OETestCase):
@OETestDepends(['OETestExample.test_example'])
diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
index c7d6db3..316f90f 100644
--- a/meta/lib/oeqa/core/context.py
+++ b/meta/lib/oeqa/core/context.py
@@ -20,11 +20,11 @@ class OETestContext(object):
files_dir = os.path.abspath(os.path.join(os.path.dirname(
os.path.abspath(__file__)), "../files"))
- def __init__(self, d=None, logger=None):
- if not type(d) is dict:
- raise TypeError("d isn't dictionary type")
+ def __init__(self, td=None, logger=None):
+ if not type(td) is dict:
+ raise TypeError("td isn't dictionary type")
- self.d = d
+ self.td = td
self.logger = logger
self._registry = {}
self._registry['cases'] = collections.OrderedDict()
@@ -148,7 +148,7 @@ class OETestContextExecutor(object):
default_cases = [os.path.join(os.path.abspath(os.path.dirname(__file__)),
'cases/example')]
- default_data = os.path.join(default_cases[0], 'data.json')
+ default_test_data = os.path.join(default_cases[0], 'data.json')
def register_commands(self, logger, subparsers):
self.parser = subparsers.add_parser(self.name, help=self.help,
@@ -160,12 +160,12 @@ class OETestContextExecutor(object):
default=self.default_output_log,
help="results output log, default: %s" % self.default_output_log)
- if self.default_data:
- self.parser.add_argument('--data-file', action='store',
- default=self.default_data,
- help="data file to load, default: %s" % self.default_data)
+ if self.default_test_data:
+ self.parser.add_argument('--test-data-file', action='store',
+ default=self.default_test_data,
+ help="data file to load, default: %s" % self.default_test_data)
else:
- self.parser.add_argument('--data-file', action='store',
+ self.parser.add_argument('--test-data-file', action='store',
help="data file to load")
if self.default_cases:
@@ -197,11 +197,11 @@ class OETestContextExecutor(object):
self.tc_kwargs['run'] = {}
self.tc_kwargs['init']['logger'] = self._setup_logger(logger, args)
- if args.data_file:
- self.tc_kwargs['init']['d'] = json.load(
- open(args.data_file, "r"))
+ if args.test_data_file:
+ self.tc_kwargs['init']['td'] = json.load(
+ open(args.test_data_file, "r"))
else:
- self.tc_kwargs['init']['d'] = {}
+ self.tc_kwargs['init']['td'] = {}
self.module_paths = args.CASES_PATHS
diff --git a/meta/lib/oeqa/core/decorator/data.py b/meta/lib/oeqa/core/decorator/data.py
index 51ef6fe..73cca88 100644
--- a/meta/lib/oeqa/core/decorator/data.py
+++ b/meta/lib/oeqa/core/decorator/data.py
@@ -20,17 +20,17 @@ class skipIfDataVar(OETestDecorator):
def setUpDecorator(self):
msg = 'Checking if %r value is %r to skip test' % (self.var, self.value)
self.logger.debug(msg)
- if self.case.tc.d.get(self.var) == self.value:
+ if self.case.td.get(self.var) == self.value:
self.case.skipTest(self.msg)
@registerDecorator
class OETestDataDepends(OETestDecorator):
- attrs = ('data_depends',)
+ attrs = ('td_depends',)
def setUpDecorator(self):
- for v in self.data_depends:
+ for v in self.td_depends:
try:
- value = self.case.d[v]
+ value = self.case.td[v]
except KeyError:
raise OEQAMissingVariable("Test case need %s variable but"\
- " isn't into d" % v)
+ " isn't into td" % v)
diff --git a/meta/lib/oeqa/core/loader.py b/meta/lib/oeqa/core/loader.py
index 94f71ba..c73ef9a 100644
--- a/meta/lib/oeqa/core/loader.py
+++ b/meta/lib/oeqa/core/loader.py
@@ -66,7 +66,7 @@ class OETestLoader(unittest.TestLoader):
def _patchCaseClass(self, testCaseClass):
# Adds custom attributes to the OETestCase class
setattr(testCaseClass, 'tc', self.tc)
- setattr(testCaseClass, 'd', self.tc.d)
+ setattr(testCaseClass, 'td', self.tc.td)
setattr(testCaseClass, 'logger', self.tc.logger)
def _validateFilters(self, filters, decorator_filters):
diff --git a/meta/lib/oeqa/core/tests/cases/data.py b/meta/lib/oeqa/core/tests/cases/data.py
index 4d8fad0..88003a6 100644
--- a/meta/lib/oeqa/core/tests/cases/data.py
+++ b/meta/lib/oeqa/core/tests/cases/data.py
@@ -11,9 +11,9 @@ class DataTest(OETestCase):
@OETestDataDepends(['MACHINE',])
@OETestTag('dataTestOk')
def testDataOk(self):
- self.assertEqual(self.d.get('IMAGE'), 'core-image-minimal')
- self.assertEqual(self.d.get('ARCH'), 'x86')
- self.assertEqual(self.d.get('MACHINE'), 'qemuarm')
+ self.assertEqual(self.td.get('IMAGE'), 'core-image-minimal')
+ self.assertEqual(self.td.get('ARCH'), 'x86')
+ self.assertEqual(self.td.get('MACHINE'), 'qemuarm')
@OETestTag('dataTestFail')
def testDataFail(self):
--
2.1.4
^ permalink raw reply related
* [PATCH 17/32] oeqa/utils/path: Add remove_safe function
From: Aníbal Limón @ 2016-12-06 21:44 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
Checks if path exists before try to remove of avoid exception.
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
meta/lib/oeqa/core/utils/path.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/meta/lib/oeqa/core/utils/path.py b/meta/lib/oeqa/core/utils/path.py
index cb06523..a21caad 100644
--- a/meta/lib/oeqa/core/utils/path.py
+++ b/meta/lib/oeqa/core/utils/path.py
@@ -12,3 +12,8 @@ def findFile(file_name, directory):
if file_name in f:
return os.path.join(r, file_name)
return None
+
+def remove_safe(path):
+ if os.path.exists(path):
+ os.remove(path)
+
--
2.1.4
^ permalink raw reply related
* [PATCH 14/32] classes/rootfs-postcommands: Add write_image_test_data
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
This function will generates testdata.json by image type.
[YOCTO #10231]
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
meta/classes/rootfs-postcommands.bbclass | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/meta/classes/rootfs-postcommands.bbclass b/meta/classes/rootfs-postcommands.bbclass
index 0c7ceea..5cfd2fe 100644
--- a/meta/classes/rootfs-postcommands.bbclass
+++ b/meta/classes/rootfs-postcommands.bbclass
@@ -14,6 +14,9 @@ ROOTFS_POSTPROCESS_COMMAND += "rootfs_update_timestamp ; "
# Tweak the mount options for rootfs in /etc/fstab if read-only-rootfs is enabled
ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", "read_only_rootfs_hook; ", "",d)}'
+# Generates test data file with data store variables expanded in json format
+ROOTFS_POSTPROCESS_COMMAND += "write_image_test_data ; "
+
# Write manifest
IMAGE_MANIFEST = "${IMGDEPLOYDIR}/${IMAGE_NAME}.rootfs.manifest"
ROOTFS_POSTUNINSTALL_COMMAND =+ "write_image_manifest ; "
@@ -278,3 +281,18 @@ rootfs_check_host_user_contaminated () {
rootfs_sysroot_relativelinks () {
sysroot-relativelinks.py ${SDK_OUTPUT}/${SDKTARGETSYSROOT}
}
+
+# Generated test data json file
+python write_image_test_data() {
+ from oe.data import export2json
+
+ testdata = "%s/%s.testdata.json" % (d.getVar('DEPLOY_DIR_IMAGE', True), d.getVar('IMAGE_NAME', True))
+ testdata_link = "%s/%s.testdata.json" % (d.getVar('DEPLOY_DIR_IMAGE', True), d.getVar('IMAGE_LINK_NAME', True))
+
+ bb.utils.mkdirhier(os.path.dirname(testdata))
+ export2json(d, testdata)
+
+ if os.path.lexists(testdata_link):
+ os.remove(testdata_link)
+ os.symlink(os.path.basename(testdata), testdata_link)
+}
--
2.1.4
^ permalink raw reply related
* [PATCH 15/32] classes/populate_sdk_base: Add write_sdk_test_data to postprocess
From: Aníbal Limón @ 2016-12-06 21:44 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
This function will generates testdata.json per SDK type.
[YOCTO #10231]
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
meta/classes/populate_sdk_base.bbclass | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass
index 220cde6..f0fc089 100644
--- a/meta/classes/populate_sdk_base.bbclass
+++ b/meta/classes/populate_sdk_base.bbclass
@@ -73,6 +73,13 @@ python write_target_sdk_manifest () {
output.write(format_pkg_list(pkgs, 'ver'))
}
+python write_sdk_test_data() {
+ from oe.data import export2json
+ testdata = "%s/%s.testdata.json" % (d.getVar('SDKDEPLOYDIR', True), d.getVar('TOOLCHAIN_OUTPUTNAME', True))
+ bb.utils.mkdirhier(os.path.dirname(testdata))
+ export2json(d, testdata)
+}
+
python write_host_sdk_manifest () {
from oe.sdk import sdk_list_installed_packages
from oe.utils import format_pkg_list
@@ -84,7 +91,7 @@ python write_host_sdk_manifest () {
output.write(format_pkg_list(pkgs, 'ver'))
}
-POPULATE_SDK_POST_TARGET_COMMAND_append = " write_target_sdk_manifest ; "
+POPULATE_SDK_POST_TARGET_COMMAND_append = " write_target_sdk_manifest ; write_sdk_test_data ; "
POPULATE_SDK_POST_HOST_COMMAND_append = " write_host_sdk_manifest; "
SDK_PACKAGING_COMMAND = "${@'${SDK_PACKAGING_FUNC};' if '${SDK_PACKAGING_FUNC}' else ''}"
SDK_POSTPROCESS_COMMAND = " create_sdk_files; check_sdk_sysroots; tar_sdk; ${SDK_PACKAGING_COMMAND} "
--
2.1.4
^ permalink raw reply related
* [PATCH 06/32] oeqa/core/decorator: Add support for OETimeout decorator
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
From: Mariano Lopez <mariano.lopez@linux.intel.com>
The OETimeout provides support for specify certain timeout
in seconds for a test case, if the timeout is reach the SIGALRM
is sent and an exception is raised to notify the timeout.
[YOCTO #10235]
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
meta/lib/oeqa/core/decorator/oetimeout.py | 25 +++++++++++++++++++++++++
1 file changed, 25 insertions(+)
create mode 100644 meta/lib/oeqa/core/decorator/oetimeout.py
diff --git a/meta/lib/oeqa/core/decorator/oetimeout.py b/meta/lib/oeqa/core/decorator/oetimeout.py
new file mode 100644
index 0000000..a247583
--- /dev/null
+++ b/meta/lib/oeqa/core/decorator/oetimeout.py
@@ -0,0 +1,25 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import signal
+from . import OETestDecorator, registerDecorator
+from oeqa.core.exception import OEQATimeoutError
+
+@registerDecorator
+class OETimeout(OETestDecorator):
+ attrs = ('oetimeout',)
+
+ def setUpDecorator(self):
+ timeout = self.oetimeout
+ def _timeoutHandler(signum, frame):
+ raise OEQATimeoutError("Timed out after %s "
+ "seconds of execution" % timeout)
+
+ self.logger.debug("Setting up a %d second(s) timeout" % self.oetimeout)
+ self.alarmSignal = signal.signal(signal.SIGALRM, _timeoutHandler)
+ signal.alarm(self.oetimeout)
+
+ def tearDownDecorator(self):
+ signal.alarm(0)
+ signal.signal(signal.SIGALRM, self.alarmSignal)
+ self.logger.debug("Removed SIGALRM handler")
--
2.1.4
^ permalink raw reply related
* [PATCH 13/32] oe/data: Add export2json function
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
The export2json function export the variables contained in
the data store to JSON format, the main usage for now will be
to provide test data to QA framework.
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
meta/lib/oe/data.py | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/meta/lib/oe/data.py b/meta/lib/oe/data.py
index ee48950..db3bd6d 100644
--- a/meta/lib/oe/data.py
+++ b/meta/lib/oe/data.py
@@ -1,3 +1,4 @@
+import json
import oe.maketype
def typed_value(key, d):
@@ -15,3 +16,30 @@ def typed_value(key, d):
return oe.maketype.create(d.getVar(key, True) or '', var_type, **flags)
except (TypeError, ValueError) as exc:
bb.msg.fatal("Data", "%s: %s" % (key, str(exc)))
+
+def export2json(d, json_file, expand=True):
+ data2export = {}
+ keys2export = []
+
+ for key in d.keys():
+ if key.startswith("_"):
+ continue
+ elif key.startswith("BB"):
+ continue
+ elif key.startswith("B_pn"):
+ continue
+ elif key.startswith("do_"):
+ continue
+ elif d.getVarFlag(key, "func", True):
+ continue
+
+ keys2export.append(key)
+
+ for key in keys2export:
+ try:
+ data2export[key] = d.getVar(key, expand)
+ except bb.data_smart.ExpansionError:
+ data2export[key] = ''
+
+ with open(json_file, "w") as f:
+ json.dump(data2export, f, skipkeys=True, indent=4, sort_keys=True)
--
2.1.4
^ permalink raw reply related
* [PATCH 12/32] oeqa/core: Add README
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
The README has an introduction and explains how to run the test suite
and creates a new Test component.
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
meta/lib/oeqa/core/README | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
create mode 100644 meta/lib/oeqa/core/README
diff --git a/meta/lib/oeqa/core/README b/meta/lib/oeqa/core/README
new file mode 100644
index 0000000..0c859fd
--- /dev/null
+++ b/meta/lib/oeqa/core/README
@@ -0,0 +1,38 @@
+= OEQA Framework =
+
+== Introduction ==
+
+This is the new OEQA framework the base clases of the framework
+are in this module oeqa/core the subsequent components needs to
+extend this classes.
+
+A new/unique runner was created called oe-test and is under scripts/
+oe-test, this new runner scans over oeqa module searching for test
+components that supports OETestContextExecutor implemented in context
+module (i.e. oeqa/core/context.py).
+
+For execute an example:
+
+$ source oe-init-build-env
+$ oe-test core
+
+For list supported components:
+
+$ oe-test -h
+
+== Create new Test component ==
+
+Usally for add a new Test component the developer needs to extend
+OETestContext/OETestContextExecutor in context.py and OETestCase in
+case.py.
+
+== How to run the testing of the OEQA framework ==
+
+Run all tests:
+
+$ PATH=$PATH:../../ python3 -m unittest discover -s tests
+
+Run some test:
+
+$ cd tests/
+$ ./test_data.py
--
2.1.4
^ permalink raw reply related
* [PATCH 11/32] oeqa/core/cases: Add example test cases
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
Serves as an first input of how to the OEQA framework works.
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
meta/lib/oeqa/core/cases/__init__.py | 0
meta/lib/oeqa/core/cases/example/data.json | 1 +
meta/lib/oeqa/core/cases/example/test_basic.py | 20 ++++++++++++++++++++
3 files changed, 21 insertions(+)
create mode 100644 meta/lib/oeqa/core/cases/__init__.py
create mode 100644 meta/lib/oeqa/core/cases/example/data.json
create mode 100644 meta/lib/oeqa/core/cases/example/test_basic.py
diff --git a/meta/lib/oeqa/core/cases/__init__.py b/meta/lib/oeqa/core/cases/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/meta/lib/oeqa/core/cases/example/data.json b/meta/lib/oeqa/core/cases/example/data.json
new file mode 100644
index 0000000..21d6b16
--- /dev/null
+++ b/meta/lib/oeqa/core/cases/example/data.json
@@ -0,0 +1 @@
+{"ARCH": "x86", "IMAGE": "core-image-minimal"}
\ No newline at end of file
diff --git a/meta/lib/oeqa/core/cases/example/test_basic.py b/meta/lib/oeqa/core/cases/example/test_basic.py
new file mode 100644
index 0000000..8b404fe
--- /dev/null
+++ b/meta/lib/oeqa/core/cases/example/test_basic.py
@@ -0,0 +1,20 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.depends import OETestDepends
+
+class OETestExample(OETestCase):
+ def test_example(self):
+ self.logger.info('IMAGE: %s' % self.d.get('IMAGE'))
+ self.assertEqual('core-image-minimal', self.d.get('IMAGE'))
+ self.logger.info('ARCH: %s' % self.d.get('ARCH'))
+ self.assertEqual('x86', self.d.get('ARCH'))
+
+class OETestExampleDepend(OETestCase):
+ @OETestDepends(['OETestExample.test_example'])
+ def test_example_depends(self):
+ pass
+
+ def test_example_no_depends(self):
+ pass
--
2.1.4
^ permalink raw reply related
* [PATCH 08/32] oeqa/core: Add tests for the OEQA framework
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
From: Mariano Lopez <mariano.lopez@linux.intel.com>
This test suite covers the current functionality for the OEQA
framework.
For run certain test suite,
$ cd meta/lib/oeqa/core/tests
$ ./test_data.py
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
meta/lib/oeqa/core/tests/__init__.py | 0
meta/lib/oeqa/core/tests/cases/data.py | 20 +++
meta/lib/oeqa/core/tests/cases/depends.py | 38 ++++++
.../oeqa/core/tests/cases/loader/invalid/oeid.py | 15 +++
.../oeqa/core/tests/cases/loader/valid/another.py | 9 ++
meta/lib/oeqa/core/tests/cases/oeid.py | 18 +++
meta/lib/oeqa/core/tests/cases/oetag.py | 18 +++
meta/lib/oeqa/core/tests/cases/timeout.py | 18 +++
meta/lib/oeqa/core/tests/common.py | 35 ++++++
meta/lib/oeqa/core/tests/test_data.py | 51 ++++++++
meta/lib/oeqa/core/tests/test_decorators.py | 135 +++++++++++++++++++++
meta/lib/oeqa/core/tests/test_loader.py | 86 +++++++++++++
meta/lib/oeqa/core/tests/test_runner.py | 38 ++++++
13 files changed, 481 insertions(+)
create mode 100644 meta/lib/oeqa/core/tests/__init__.py
create mode 100644 meta/lib/oeqa/core/tests/cases/data.py
create mode 100644 meta/lib/oeqa/core/tests/cases/depends.py
create mode 100644 meta/lib/oeqa/core/tests/cases/loader/invalid/oeid.py
create mode 100644 meta/lib/oeqa/core/tests/cases/loader/valid/another.py
create mode 100644 meta/lib/oeqa/core/tests/cases/oeid.py
create mode 100644 meta/lib/oeqa/core/tests/cases/oetag.py
create mode 100644 meta/lib/oeqa/core/tests/cases/timeout.py
create mode 100644 meta/lib/oeqa/core/tests/common.py
create mode 100755 meta/lib/oeqa/core/tests/test_data.py
create mode 100755 meta/lib/oeqa/core/tests/test_decorators.py
create mode 100755 meta/lib/oeqa/core/tests/test_loader.py
create mode 100755 meta/lib/oeqa/core/tests/test_runner.py
diff --git a/meta/lib/oeqa/core/tests/__init__.py b/meta/lib/oeqa/core/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/meta/lib/oeqa/core/tests/cases/data.py b/meta/lib/oeqa/core/tests/cases/data.py
new file mode 100644
index 0000000..4d8fad0
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/cases/data.py
@@ -0,0 +1,20 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.oetag import OETestTag
+from oeqa.core.decorator.data import OETestDataDepends
+
+class DataTest(OETestCase):
+ data_vars = ['IMAGE', 'ARCH']
+
+ @OETestDataDepends(['MACHINE',])
+ @OETestTag('dataTestOk')
+ def testDataOk(self):
+ self.assertEqual(self.d.get('IMAGE'), 'core-image-minimal')
+ self.assertEqual(self.d.get('ARCH'), 'x86')
+ self.assertEqual(self.d.get('MACHINE'), 'qemuarm')
+
+ @OETestTag('dataTestFail')
+ def testDataFail(self):
+ pass
diff --git a/meta/lib/oeqa/core/tests/cases/depends.py b/meta/lib/oeqa/core/tests/cases/depends.py
new file mode 100644
index 0000000..17cdd90
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/cases/depends.py
@@ -0,0 +1,38 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.depends import OETestDepends
+
+class DependsTest(OETestCase):
+
+ def testDependsFirst(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestDepends(['testDependsFirst'])
+ def testDependsSecond(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestDepends(['testDependsSecond'])
+ def testDependsThird(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestDepends(['testDependsSecond'])
+ def testDependsFourth(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestDepends(['testDependsThird', 'testDependsFourth'])
+ def testDependsFifth(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestDepends(['testDependsCircular3'])
+ def testDependsCircular1(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestDepends(['testDependsCircular1'])
+ def testDependsCircular2(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestDepends(['testDependsCircular2'])
+ def testDependsCircular3(self):
+ self.assertTrue(True, msg='How is this possible?')
diff --git a/meta/lib/oeqa/core/tests/cases/loader/invalid/oeid.py b/meta/lib/oeqa/core/tests/cases/loader/invalid/oeid.py
new file mode 100644
index 0000000..038d445
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/cases/loader/invalid/oeid.py
@@ -0,0 +1,15 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+
+class AnotherIDTest(OETestCase):
+
+ def testAnotherIdGood(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ def testAnotherIdOther(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ def testAnotherIdNone(self):
+ self.assertTrue(True, msg='How is this possible?')
diff --git a/meta/lib/oeqa/core/tests/cases/loader/valid/another.py b/meta/lib/oeqa/core/tests/cases/loader/valid/another.py
new file mode 100644
index 0000000..c9ffd17
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/cases/loader/valid/another.py
@@ -0,0 +1,9 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+
+class AnotherTest(OETestCase):
+
+ def testAnother(self):
+ self.assertTrue(True, msg='How is this possible?')
diff --git a/meta/lib/oeqa/core/tests/cases/oeid.py b/meta/lib/oeqa/core/tests/cases/oeid.py
new file mode 100644
index 0000000..c2d3d32
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/cases/oeid.py
@@ -0,0 +1,18 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.oeid import OETestID
+
+class IDTest(OETestCase):
+
+ @OETestID(101)
+ def testIdGood(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestID(102)
+ def testIdOther(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ def testIdNone(self):
+ self.assertTrue(True, msg='How is this possible?')
diff --git a/meta/lib/oeqa/core/tests/cases/oetag.py b/meta/lib/oeqa/core/tests/cases/oetag.py
new file mode 100644
index 0000000..0cae02e
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/cases/oetag.py
@@ -0,0 +1,18 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.oetag import OETestTag
+
+class TagTest(OETestCase):
+
+ @OETestTag('goodTag')
+ def testTagGood(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestTag('otherTag')
+ def testTagOther(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ def testTagNone(self):
+ self.assertTrue(True, msg='How is this possible?')
diff --git a/meta/lib/oeqa/core/tests/cases/timeout.py b/meta/lib/oeqa/core/tests/cases/timeout.py
new file mode 100644
index 0000000..870c3157
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/cases/timeout.py
@@ -0,0 +1,18 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from time import sleep
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.oetimeout import OETimeout
+
+class TimeoutTest(OETestCase):
+
+ @OETimeout(1)
+ def testTimeoutPass(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETimeout(1)
+ def testTimeoutFail(self):
+ sleep(2)
+ self.assertTrue(True, msg='How is this possible?')
diff --git a/meta/lib/oeqa/core/tests/common.py b/meta/lib/oeqa/core/tests/common.py
new file mode 100644
index 0000000..52b18a1
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/common.py
@@ -0,0 +1,35 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import sys
+import os
+
+import unittest
+import logging
+import os
+
+logger = logging.getLogger("oeqa")
+logger.setLevel(logging.INFO)
+consoleHandler = logging.StreamHandler()
+formatter = logging.Formatter('OEQATest: %(message)s')
+consoleHandler.setFormatter(formatter)
+logger.addHandler(consoleHandler)
+
+def setup_sys_path():
+ directory = os.path.dirname(os.path.abspath(__file__))
+ oeqa_lib = os.path.realpath(os.path.join(directory, '../../../'))
+ if not oeqa_lib in sys.path:
+ sys.path.insert(0, oeqa_lib)
+
+class TestBase(unittest.TestCase):
+ def setUp(self):
+ self.logger = logger
+ directory = os.path.dirname(os.path.abspath(__file__))
+ self.cases_path = os.path.join(directory, 'cases')
+
+ def _testLoader(self, d={}, modules=[], tests=[], filters={}):
+ from oeqa.core.context import OETestContext
+ tc = OETestContext(d, self.logger)
+ tc.loadTests(self.cases_path, modules=modules, tests=tests,
+ filters=filters)
+ return tc
diff --git a/meta/lib/oeqa/core/tests/test_data.py b/meta/lib/oeqa/core/tests/test_data.py
new file mode 100755
index 0000000..320468c
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/test_data.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import unittest
+import logging
+import os
+
+from common import setup_sys_path, TestBase
+setup_sys_path()
+
+from oeqa.core.exception import OEQAMissingVariable
+from oeqa.core.utils.test import getCaseMethod, getSuiteCasesNames
+
+class TestData(TestBase):
+ modules = ['data']
+
+ def test_data_fail_missing_variable(self):
+ expectedException = "oeqa.core.exception.OEQAMissingVariable"
+
+ tc = self._testLoader(modules=self.modules)
+ self.assertEqual(False, tc.runTests().wasSuccessful())
+ for test, data in tc._results['errors']:
+ expect = False
+ if expectedException in data:
+ expect = True
+
+ self.assertTrue(expect)
+
+ def test_data_fail_wrong_variable(self):
+ expectedError = 'AssertionError'
+ d = {'IMAGE' : 'core-image-sato', 'ARCH' : 'arm'}
+
+ tc = self._testLoader(d=d, modules=self.modules)
+ self.assertEqual(False, tc.runTests().wasSuccessful())
+ for test, data in tc._results['failures']:
+ expect = False
+ if expectedError in data:
+ expect = True
+
+ self.assertTrue(expect)
+
+ def test_data_ok(self):
+ d = {'IMAGE' : 'core-image-minimal', 'ARCH' : 'x86', 'MACHINE' : 'qemuarm'}
+
+ tc = self._testLoader(d=d, modules=self.modules)
+ self.assertEqual(True, tc.runTests().wasSuccessful())
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/meta/lib/oeqa/core/tests/test_decorators.py b/meta/lib/oeqa/core/tests/test_decorators.py
new file mode 100755
index 0000000..f7d11e8
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/test_decorators.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import signal
+import unittest
+
+from common import setup_sys_path, TestBase
+setup_sys_path()
+
+from oeqa.core.exception import OEQADependency
+from oeqa.core.utils.test import getCaseMethod, getSuiteCasesNames, getSuiteCasesIDs
+
+class TestFilterDecorator(TestBase):
+
+ def _runFilterTest(self, modules, filters, expect, msg):
+ tc = self._testLoader(modules=modules, filters=filters)
+ test_loaded = set(getSuiteCasesNames(tc.suites))
+ self.assertEqual(expect, test_loaded, msg=msg)
+
+ def test_oetag(self):
+ # Get all cases without filtering.
+ filter_all = {}
+ test_all = {'testTagGood', 'testTagOther', 'testTagNone'}
+ msg_all = 'Failed to get all oetag cases without filtering.'
+
+ # Get cases with 'goodTag'.
+ filter_good = {'oetag':'goodTag'}
+ test_good = {'testTagGood'}
+ msg_good = 'Failed to get just one test filtering with "goodTag" oetag.'
+
+ # Get cases with an invalid tag.
+ filter_invalid = {'oetag':'invalidTag'}
+ test_invalid = set()
+ msg_invalid = 'Failed to filter all test using an invalid oetag.'
+
+ tests = ((filter_all, test_all, msg_all),
+ (filter_good, test_good, msg_good),
+ (filter_invalid, test_invalid, msg_invalid))
+
+ for test in tests:
+ self._runFilterTest(['oetag'], test[0], test[1], test[2])
+
+ def test_oeid(self):
+ # Get all cases without filtering.
+ filter_all = {}
+ test_all = {'testIdGood', 'testIdOther', 'testIdNone'}
+ msg_all = 'Failed to get all oeid cases without filtering.'
+
+ # Get cases with '101' oeid.
+ filter_good = {'oeid': 101}
+ test_good = {'testIdGood'}
+ msg_good = 'Failed to get just one tes filtering with "101" oeid.'
+
+ # Get cases with an invalid id.
+ filter_invalid = {'oeid':999}
+ test_invalid = set()
+ msg_invalid = 'Failed to filter all test using an invalid oeid.'
+
+ tests = ((filter_all, test_all, msg_all),
+ (filter_good, test_good, msg_good),
+ (filter_invalid, test_invalid, msg_invalid))
+
+ for test in tests:
+ self._runFilterTest(['oeid'], test[0], test[1], test[2])
+
+class TestDependsDecorator(TestBase):
+ modules = ['depends']
+
+ def test_depends_order(self):
+ tests = ['depends.DependsTest.testDependsFirst',
+ 'depends.DependsTest.testDependsSecond',
+ 'depends.DependsTest.testDependsThird',
+ 'depends.DependsTest.testDependsFourth',
+ 'depends.DependsTest.testDependsFifth']
+ tests2 = list(tests)
+ tests2[2], tests2[3] = tests[3], tests[2]
+ tc = self._testLoader(modules=self.modules, tests=tests)
+ test_loaded = getSuiteCasesIDs(tc.suites)
+ result = True if test_loaded == tests or test_loaded == tests2 else False
+ msg = 'Failed to order tests using OETestDepends decorator.\nTest order:'\
+ ' %s.\nExpected: %s\nOr: %s' % (test_loaded, tests, tests2)
+ self.assertTrue(result, msg=msg)
+
+ def test_depends_fail_missing_dependency(self):
+ expect = "TestCase depends.DependsTest.testDependsSecond depends on "\
+ "depends.DependsTest.testDependsFirst and isn't available"
+ tests = ['depends.DependsTest.testDependsSecond']
+ try:
+ # Must throw OEQADependency because missing 'testDependsFirst'
+ tc = self._testLoader(modules=self.modules, tests=tests)
+ self.fail('Expected OEQADependency exception')
+ except OEQADependency as e:
+ result = True if expect in str(e) else False
+ msg = 'Expected OEQADependency exception missing testDependsFirst test'
+ self.assertTrue(result, msg=msg)
+
+ def test_depends_fail_circular_dependency(self):
+ expect = 'have a circular dependency'
+ tests = ['depends.DependsTest.testDependsCircular1',
+ 'depends.DependsTest.testDependsCircular2',
+ 'depends.DependsTest.testDependsCircular3']
+ try:
+ # Must throw OEQADependency because circular dependency
+ tc = self._testLoader(modules=self.modules, tests=tests)
+ self.fail('Expected OEQADependency exception')
+ except OEQADependency as e:
+ result = True if expect in str(e) else False
+ msg = 'Expected OEQADependency exception having a circular dependency'
+ self.assertTrue(result, msg=msg)
+
+class TestTimeoutDecorator(TestBase):
+ modules = ['timeout']
+
+ def test_timeout(self):
+ tests = ['timeout.TimeoutTest.testTimeoutPass']
+ msg = 'Failed to run test using OETestTimeout'
+ alarm_signal = signal.getsignal(signal.SIGALRM)
+ tc = self._testLoader(modules=self.modules, tests=tests)
+ self.assertTrue(tc.runTests().wasSuccessful(), msg=msg)
+ msg = "OETestTimeout didn't restore SIGALRM"
+ self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg)
+
+ def test_timeout_fail(self):
+ tests = ['timeout.TimeoutTest.testTimeoutFail']
+ msg = "OETestTimeout test didn't timeout as expected"
+ alarm_signal = signal.getsignal(signal.SIGALRM)
+ tc = self._testLoader(modules=self.modules, tests=tests)
+ self.assertFalse(tc.runTests().wasSuccessful(), msg=msg)
+ msg = "OETestTimeout didn't restore SIGALRM"
+ self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/meta/lib/oeqa/core/tests/test_loader.py b/meta/lib/oeqa/core/tests/test_loader.py
new file mode 100755
index 0000000..b79b8ba
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/test_loader.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import unittest
+
+from common import setup_sys_path, TestBase
+setup_sys_path()
+
+from oeqa.core.exception import OEQADependency
+from oeqa.core.utils.test import getSuiteModules, getSuiteCasesIDs
+
+class TestLoader(TestBase):
+
+ def test_fail_empty_filter(self):
+ filters = {'oetag' : ''}
+ expect = 'Filter oetag specified is empty'
+ msg = 'Expected TypeError exception for having invalid filter'
+ try:
+ # Must throw TypeError because empty filter
+ tc = self._testLoader(filters=filters)
+ self.fail(msg)
+ except TypeError as e:
+ result = True if expect in str(e) else False
+ self.assertTrue(result, msg=msg)
+
+ def test_fail_invalid_filter(self):
+ filters = {'invalid' : 'good'}
+ expect = 'filter but not declared in any of'
+ msg = 'Expected TypeError exception for having invalid filter'
+ try:
+ # Must throw TypeError because invalid filter
+ tc = self._testLoader(filters=filters)
+ self.fail(msg)
+ except TypeError as e:
+ result = True if expect in str(e) else False
+ self.assertTrue(result, msg=msg)
+
+ def test_fail_duplicated_module(self):
+ cases_path = self.cases_path
+ invalid_path = os.path.join(cases_path, 'loader', 'invalid')
+ self.cases_path = [self.cases_path, invalid_path]
+ expect = 'Duplicated oeid module found in'
+ msg = 'Expected ImportError exception for having duplicated module'
+ try:
+ # Must throw ImportEror because duplicated module
+ tc = self._testLoader()
+ self.fail(msg)
+ except ImportError as e:
+ result = True if expect in str(e) else False
+ self.assertTrue(result, msg=msg)
+ finally:
+ self.cases_path = cases_path
+
+ def test_filter_modules(self):
+ expected_modules = {'oeid', 'oetag'}
+ tc = self._testLoader(modules=expected_modules)
+ modules = getSuiteModules(tc.suites)
+ msg = 'Expected just %s modules' % ', '.join(expected_modules)
+ self.assertEqual(modules, expected_modules, msg=msg)
+
+ def test_filter_cases(self):
+ modules = ['oeid', 'oetag', 'data']
+ expected_cases = {'data.DataTest.testDataOk',
+ 'oetag.TagTest.testTagGood',
+ 'oeid.IDTest.testIdGood'}
+ tc = self._testLoader(modules=modules, tests=expected_cases)
+ cases = set(getSuiteCasesIDs(tc.suites))
+ msg = 'Expected just %s cases' % ', '.join(expected_cases)
+ self.assertEqual(cases, expected_cases, msg=msg)
+
+ def test_import_from_paths(self):
+ cases_path = self.cases_path
+ cases2_path = os.path.join(cases_path, 'loader', 'valid')
+ expected_modules = {'oeid', 'another'}
+ self.cases_path = [self.cases_path, cases2_path]
+ tc = self._testLoader(modules=expected_modules)
+ modules = getSuiteModules(tc.suites)
+ self.cases_path = cases_path
+ msg = 'Expected modules from two different paths'
+ self.assertEqual(modules, expected_modules, msg=msg)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/meta/lib/oeqa/core/tests/test_runner.py b/meta/lib/oeqa/core/tests/test_runner.py
new file mode 100755
index 0000000..a3f3861
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/test_runner.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import unittest
+import logging
+import tempfile
+
+from common import setup_sys_path, TestBase
+setup_sys_path()
+
+from oeqa.core.runner import OEStreamLogger
+
+class TestRunner(TestBase):
+ def test_stream_logger(self):
+ fp = tempfile.TemporaryFile(mode='w+')
+
+ logging.basicConfig(format='%(message)s', stream=fp)
+ logger = logging.getLogger()
+ logger.setLevel(logging.INFO)
+
+ oeSL = OEStreamLogger(logger)
+
+ lines = ['init', 'bigline_' * 65535, 'morebigline_' * 65535 * 4, 'end']
+ for line in lines:
+ oeSL.write(line)
+
+ fp.seek(0)
+ fp_lines = fp.readlines()
+ for i, fp_line in enumerate(fp_lines):
+ fp_line = fp_line.strip()
+ self.assertEqual(lines[i], fp_line)
+
+ fp.close()
+
+if __name__ == '__main__':
+ unittest.main()
--
2.1.4
^ permalink raw reply related
* [PATCH 10/32] oeqa/core/context: Add support of OETestContextExecutor
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
The OETestContextExecutor class supports to use oe-test for run core
test component also is a base class for the other test components
(runtime, sdk, selftest).
Te principal functionality is to support cmdline parsing and execution
of OETestContext, the test components could extend the common options
to provide specific ones. The common options between test components
are test data file, output log and test cases path's to scan.
Also it initializes the logger to be passed to the whole OEQA framework.
[YOCTO #10230]
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
meta/lib/oeqa/core/context.py | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
index d5caf53..c7d6db3 100644
--- a/meta/lib/oeqa/core/context.py
+++ b/meta/lib/oeqa/core/context.py
@@ -148,7 +148,7 @@ class OETestContextExecutor(object):
default_cases = [os.path.join(os.path.abspath(os.path.dirname(__file__)),
'cases/example')]
- default_test_data = os.path.join(default_cases[0], 'data.json')
+ default_data = os.path.join(default_cases[0], 'data.json')
def register_commands(self, logger, subparsers):
self.parser = subparsers.add_parser(self.name, help=self.help,
@@ -160,12 +160,12 @@ class OETestContextExecutor(object):
default=self.default_output_log,
help="results output log, default: %s" % self.default_output_log)
- if self.default_test_data:
- self.parser.add_argument('--test-data-file', action='store',
- default=self.default_test_data,
- help="data file to load, default: %s" % self.default_test_data)
+ if self.default_data:
+ self.parser.add_argument('--data-file', action='store',
+ default=self.default_data,
+ help="data file to load, default: %s" % self.default_data)
else:
- self.parser.add_argument('--test-data-file', action='store',
+ self.parser.add_argument('--data-file', action='store',
help="data file to load")
if self.default_cases:
@@ -197,11 +197,11 @@ class OETestContextExecutor(object):
self.tc_kwargs['run'] = {}
self.tc_kwargs['init']['logger'] = self._setup_logger(logger, args)
- if args.test_data_file:
- self.tc_kwargs['init']['td'] = json.load(
- open(args.test_data_file, "r"))
+ if args.data_file:
+ self.tc_kwargs['init']['d'] = json.load(
+ open(args.data_file, "r"))
else:
- self.tc_kwargs['init']['td'] = {}
+ self.tc_kwargs['init']['d'] = {}
self.module_paths = args.CASES_PATHS
--
2.1.4
^ permalink raw reply related
* [PATCH 09/32] scripts/oe-test: Add new oe-test script
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
The new oe-test script will be use to run test components with
one single script.
The oe-test script search for test components inside meta/lib/oeqa,
the test components needs to implement OETestContextExecutor inside
context module in order to be supported by oe-test.
[YOCTO #10230]
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
scripts/oe-test | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 98 insertions(+)
create mode 100755 scripts/oe-test
diff --git a/scripts/oe-test b/scripts/oe-test
new file mode 100755
index 0000000..9222dea
--- /dev/null
+++ b/scripts/oe-test
@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+# OpenEmbedded test tool
+#
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import sys
+import argparse
+import importlib
+import logging
+
+scripts_path = os.path.dirname(os.path.realpath(__file__))
+lib_path = scripts_path + '/lib'
+sys.path = sys.path + [lib_path]
+import argparse_oe
+import scriptutils
+import scriptpath
+scriptpath.add_oe_lib_path()
+
+from oeqa.core.context import OETestContextExecutor
+
+logger = scriptutils.logger_create('oe-test')
+
+def _load_test_components(logger):
+ components = {}
+
+ for path in sys.path:
+ base_dir = os.path.join(path, 'oeqa')
+ if os.path.exists(base_dir) and os.path.isdir(base_dir):
+ for file in os.listdir(base_dir):
+ comp_name = file
+ comp_context = os.path.join(base_dir, file, 'context.py')
+ if os.path.exists(comp_context):
+ comp_plugin = importlib.import_module('oeqa.%s.%s' % \
+ (comp_name, 'context'))
+ try:
+ if not issubclass(comp_plugin._executor_class,
+ OETestContextExecutor):
+ raise TypeError("Component %s in %s, _executor_class "\
+ "isn't derived from OETestContextExecutor."\
+ % (comp_name, comp_context))
+
+ components[comp_name] = comp_plugin._executor_class()
+ except AttributeError:
+ raise AttributeError("Component %s in %s don't have "\
+ "_executor_class defined." % (comp_name, comp_context))
+
+ return components
+
+def main():
+ parser = argparse_oe.ArgumentParser(description="OpenEmbedded test tool",
+ add_help=False,
+ epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
+ parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
+ parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
+ global_args, unparsed_args = parser.parse_known_args()
+
+ # Help is added here rather than via add_help=True, as we don't want it to
+ # be handled by parse_known_args()
+ parser.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS,
+ help='show this help message and exit')
+
+ if global_args.debug:
+ logger.setLevel(logging.DEBUG)
+ elif global_args.quiet:
+ logger.setLevel(logging.ERROR)
+
+ components = _load_test_components(logger)
+
+ subparsers = parser.add_subparsers(dest="subparser_name", title='subcommands', metavar='<subcommand>')
+ subparsers.add_subparser_group('components', 'Test components')
+ subparsers.required = True
+ for comp_name in sorted(components.keys()):
+ comp = components[comp_name]
+ comp.register_commands(logger, subparsers)
+
+ try:
+ args = parser.parse_args(unparsed_args, namespace=global_args)
+ ret = args.func(logger, args)
+ except SystemExit as err:
+ if err.code != 0:
+ raise err
+ ret = err.code
+ except argparse_oe.ArgumentUsageError as ae:
+ parser.error_subcommand(ae.message, ae.subcommand)
+
+ return ret
+
+if __name__ == '__main__':
+ try:
+ ret = main()
+ except Exception:
+ ret = 1
+ import traceback
+ traceback.print_exc()
+ sys.exit(ret)
--
2.1.4
^ permalink raw reply related
* [PATCH 07/32] oeqa/core/decorator: Add support for OETestDataDepends and skipIfDataVar
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
The OETestDataDepends decorator skips a test case if a variable
isn't into test data (d).
The skipIfDataVar decorator skips a test case if a variable
has certain value.
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
meta/lib/oeqa/core/decorator/data.py | 36 ++++++++++++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
create mode 100644 meta/lib/oeqa/core/decorator/data.py
diff --git a/meta/lib/oeqa/core/decorator/data.py b/meta/lib/oeqa/core/decorator/data.py
new file mode 100644
index 0000000..51ef6fe
--- /dev/null
+++ b/meta/lib/oeqa/core/decorator/data.py
@@ -0,0 +1,36 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.exception import OEQAMissingVariable
+
+from . import OETestDecorator, registerDecorator
+
+@registerDecorator
+class skipIfDataVar(OETestDecorator):
+ """
+ Skip test based on value of a data store's variable.
+
+ It will get the info of var from the data store and will
+ check it against value; if are equal it will skip the test
+ with msg as the reason.
+ """
+
+ attrs = ('var', 'value', 'msg')
+
+ def setUpDecorator(self):
+ msg = 'Checking if %r value is %r to skip test' % (self.var, self.value)
+ self.logger.debug(msg)
+ if self.case.tc.d.get(self.var) == self.value:
+ self.case.skipTest(self.msg)
+
+@registerDecorator
+class OETestDataDepends(OETestDecorator):
+ attrs = ('data_depends',)
+
+ def setUpDecorator(self):
+ for v in self.data_depends:
+ try:
+ value = self.case.d[v]
+ except KeyError:
+ raise OEQAMissingVariable("Test case need %s variable but"\
+ " isn't into d" % v)
--
2.1.4
^ permalink raw reply related
* [PATCH 05/32] oeqa/core/decorator: Add support for OETestID and OETestTag
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
From: Mariano Lopez <mariano.lopez@linux.intel.com>
These two decorators stores certain TAG or ID for the test case
also provides support for filtering in loading step.
[YOCTO #10236]
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
meta/lib/oeqa/core/decorator/oeid.py | 23 +++++++++++++++++++++++
meta/lib/oeqa/core/decorator/oetag.py | 24 ++++++++++++++++++++++++
2 files changed, 47 insertions(+)
create mode 100644 meta/lib/oeqa/core/decorator/oeid.py
create mode 100644 meta/lib/oeqa/core/decorator/oetag.py
diff --git a/meta/lib/oeqa/core/decorator/oeid.py b/meta/lib/oeqa/core/decorator/oeid.py
new file mode 100644
index 0000000..ea8017a
--- /dev/null
+++ b/meta/lib/oeqa/core/decorator/oeid.py
@@ -0,0 +1,23 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from . import OETestFilter, registerDecorator
+from oeqa.core.utils.misc import intToList
+
+def _idFilter(oeid, filters):
+ return False if oeid in filters else True
+
+@registerDecorator
+class OETestID(OETestFilter):
+ attrs = ('oeid',)
+
+ def bind(self, registry, case):
+ super(OETestID, self).bind(registry, case)
+
+ def filtrate(self, filters):
+ if filters.get('oeid'):
+ filterx = intToList(filters['oeid'], 'oeid')
+ del filters['oeid']
+ if _idFilter(self.oeid, filterx):
+ return True
+ return False
diff --git a/meta/lib/oeqa/core/decorator/oetag.py b/meta/lib/oeqa/core/decorator/oetag.py
new file mode 100644
index 0000000..ad38ab7
--- /dev/null
+++ b/meta/lib/oeqa/core/decorator/oetag.py
@@ -0,0 +1,24 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from . import OETestFilter, registerDecorator
+from oeqa.core.utils.misc import strToList
+
+def _tagFilter(tags, filters):
+ return False if set(tags) & set(filters) else True
+
+@registerDecorator
+class OETestTag(OETestFilter):
+ attrs = ('oetag',)
+
+ def bind(self, registry, case):
+ super(OETestTag, self).bind(registry, case)
+ self.oetag = strToList(self.oetag, 'oetag')
+
+ def filtrate(self, filters):
+ if filters.get('oetag'):
+ filterx = strToList(filters['oetag'], 'oetag')
+ del filters['oetag']
+ if _tagFilter(self.oetag, filterx):
+ return True
+ return False
--
2.1.4
^ permalink raw reply related
* [PATCH 04/32] oeqa/core/decorator: Add support for OETestDepends
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
The OETestDepends decorator could be used over test cases to
define some dependency between them.
At loading time sorting the tests to grauntee that a test case
executes before also raise an exception if found a circular
dependency between test cases.
At before test case run reviews if the dependency if meet, in the
case of don't it skips the test case run.
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
meta/lib/oeqa/core/decorator/depends.py | 94 +++++++++++++++++++++++++++++++++
1 file changed, 94 insertions(+)
create mode 100644 meta/lib/oeqa/core/decorator/depends.py
diff --git a/meta/lib/oeqa/core/decorator/depends.py b/meta/lib/oeqa/core/decorator/depends.py
new file mode 100644
index 0000000..195711c
--- /dev/null
+++ b/meta/lib/oeqa/core/decorator/depends.py
@@ -0,0 +1,94 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from unittest import SkipTest
+
+from oeqa.core.exception import OEQADependency
+
+from . import OETestDiscover, registerDecorator
+
+def _add_depends(registry, case, depends):
+ module_name = case.__module__
+ class_name = case.__class__.__name__
+
+ case_id = case.id()
+
+ for depend in depends:
+ dparts = depend.split('.')
+
+ if len(dparts) == 1:
+ depend_id = ".".join((module_name, class_name, dparts[0]))
+ elif len(dparts) == 2:
+ depend_id = ".".join((module_name, dparts[0], dparts[1]))
+ else:
+ depend_id = depend
+
+ if not case_id in registry:
+ registry[case_id] = []
+ if not depend_id in registry[case_id]:
+ registry[case_id].append(depend_id)
+
+def _validate_test_case_depends(cases, depends):
+ for case in depends:
+ if not case in cases:
+ continue
+ for dep in depends[case]:
+ if not dep in cases:
+ raise OEQADependency("TestCase %s depends on %s and isn't available"\
+ ", cases available %s." % (case, dep, str(cases.keys())))
+
+def _order_test_case_by_depends(cases, depends):
+ def _dep_resolve(graph, node, resolved, seen):
+ seen.append(node)
+ for edge in graph[node]:
+ if edge not in resolved:
+ if edge in seen:
+ raise OEQADependency("Test cases %s and %s have a circular" \
+ " dependency." % (node, edge))
+ _dep_resolve(graph, edge, resolved, seen)
+ resolved.append(node)
+
+ dep_graph = {}
+ dep_graph['__root__'] = cases.keys()
+ for case in cases:
+ if case in depends:
+ dep_graph[case] = depends[case]
+ else:
+ dep_graph[case] = []
+
+ cases_ordered = []
+ _dep_resolve(dep_graph, '__root__', cases_ordered, [])
+ cases_ordered.remove('__root__')
+
+ return [cases[case_id] for case_id in cases_ordered]
+
+def _skipTestDependency(case, depends):
+ results = case.tc._results
+ skipReasons = ['errors', 'failures', 'skipped']
+
+ for reason in skipReasons:
+ for test, _ in results[reason]:
+ if test.id() in depends:
+ raise SkipTest("Test case %s depends on %s and was in %s." \
+ % (case.id(), test.id(), reason))
+
+@registerDecorator
+class OETestDepends(OETestDiscover):
+ attrs = ('depends',)
+
+ def bind(self, registry, case):
+ super(OETestDepends, self).bind(registry, case)
+ if not registry.get('depends'):
+ registry['depends'] = {}
+ _add_depends(registry['depends'], case, self.depends)
+
+ @staticmethod
+ def discover(registry):
+ if registry.get('depends'):
+ _validate_test_case_depends(registry['cases'], registry['depends'])
+ return _order_test_case_by_depends(registry['cases'], registry['depends'])
+ else:
+ return [registry['cases'][case_id] for case_id in registry['cases']]
+
+ def setUpDecorator(self):
+ _skipTestDependency(self.case, self.depends)
--
2.1.4
^ permalink raw reply related
* [PATCH 03/32] oeqa/core: Add loader, context and decorator modules
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
loader: Implements OETestLoader handling OETestDecorator
and filtering support when load tests. The OETestLoader is
responsible to set custom methods, attrs of the OEQA
frameowork.
[YOCTO #10231]
[YOCTO #10317]
[YOCTO #10353]
decorator: Add base class OETestDecorator to provide a common
way to define decorators to be used over OETestCase's, every
decorator has a method to be called when loading tests and
before test execution starts. Special decorators could be
implemented for filter tests on loading phase.
context: Provides HIGH level API for loadTests and runTests
of certain test component (i.e. runtime, sdk, selftest).
[YOCTO #10230]
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
meta/lib/oeqa/core/context.py | 225 +++++++++++++++++++++++++++++
meta/lib/oeqa/core/decorator/__init__.py | 71 ++++++++++
meta/lib/oeqa/core/loader.py | 235 +++++++++++++++++++++++++++++++
3 files changed, 531 insertions(+)
create mode 100644 meta/lib/oeqa/core/context.py
create mode 100644 meta/lib/oeqa/core/decorator/__init__.py
create mode 100644 meta/lib/oeqa/core/loader.py
diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
new file mode 100644
index 0000000..d5caf53
--- /dev/null
+++ b/meta/lib/oeqa/core/context.py
@@ -0,0 +1,225 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import sys
+import json
+import time
+import logging
+import collections
+import re
+
+from oeqa.core.loader import OETestLoader
+from oeqa.core.runner import OETestRunner, OEStreamLogger, xmlEnabled
+
+class OETestContext(object):
+ loaderClass = OETestLoader
+ runnerClass = OETestRunner
+ streamLoggerClass = OEStreamLogger
+
+ files_dir = os.path.abspath(os.path.join(os.path.dirname(
+ os.path.abspath(__file__)), "../files"))
+
+ def __init__(self, d=None, logger=None):
+ if not type(d) is dict:
+ raise TypeError("d isn't dictionary type")
+
+ self.d = d
+ self.logger = logger
+ self._registry = {}
+ self._registry['cases'] = collections.OrderedDict()
+ self._results = {}
+
+ def _read_modules_from_manifest(self, manifest):
+ if not os.path.exists(manifest):
+ raise
+
+ modules = []
+ for line in open(manifest).readlines():
+ line = line.strip()
+ if line and not line.startswith("#"):
+ modules.append(line)
+
+ return modules
+
+ def loadTests(self, module_paths, modules=[], tests=[],
+ modules_manifest="", modules_required=[], filters={}):
+ if modules_manifest:
+ modules = self._read_modules_from_manifest(modules_manifest)
+
+ self.loader = self.loaderClass(self, module_paths, modules, tests,
+ modules_required, filters)
+ self.suites = self.loader.discover()
+
+ def runTests(self):
+ streamLogger = self.streamLoggerClass(self.logger)
+ self.runner = self.runnerClass(self, stream=streamLogger, verbosity=2)
+
+ self._run_start_time = time.time()
+ result = self.runner.run(self.suites)
+ self._run_end_time = time.time()
+
+ return result
+
+ def logSummary(self, result, component, context_msg=''):
+ self.logger.info("SUMMARY:")
+ self.logger.info("%s (%s) - Ran %d test%s in %.3fs" % (component,
+ context_msg, result.testsRun, result.testsRun != 1 and "s" or "",
+ (self._run_end_time - self._run_start_time)))
+
+ if result.wasSuccessful():
+ msg = "%s - OK - All required tests passed" % component
+ else:
+ msg = "%s - FAIL - Required tests failed" % component
+ skipped = len(self._results['skipped'])
+ if skipped:
+ msg += " (skipped=%d)" % skipped
+ self.logger.info(msg)
+
+ def _logDetailsNotPassed(self, case, type, desc):
+ found = False
+
+ for (scase, msg) in self._results[type]:
+ # XXX: When XML reporting is enabled scase is
+ # xmlrunner.result._TestInfo instance instead of
+ # string.
+ if xmlEnabled:
+ if case.id() == scase.test_id:
+ found = True
+ break
+ scase_str = scase.test_id
+ else:
+ if case == scase:
+ found = True
+ break
+ scase_str = str(scase)
+
+ # When fails at module or class level the class name is passed as string
+ # so figure out to see if match
+ m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str)
+ if m:
+ if case.__class__.__module__ == m.group('module_name'):
+ found = True
+ break
+
+ m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str)
+ if m:
+ class_name = "%s.%s" % (case.__class__.__module__,
+ case.__class__.__name__)
+
+ if class_name == m.group('class_name'):
+ found = True
+ break
+
+ if found:
+ self.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(),
+ case.oe_id if hasattr(case, 'oeid') else '-1', desc))
+ if msg:
+ self.logger.info(msg)
+
+ return found
+
+ def logDetails(self):
+ self.logger.info("RESULTS:")
+ for case_name in self._registry['cases']:
+ case = self._registry['cases'][case_name]
+
+ result_types = ['failures', 'errors', 'skipped', 'expectedFailures']
+ result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL']
+
+ match = False
+ for idx, name in enumerate(result_types):
+ match = self._logDetailsNotPassed(case, result_types[idx],
+ result_desc[idx])
+ if match:
+ break
+
+ if not match:
+ self.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(),
+ case.oe_id if hasattr(case, 'oeid') else '-1',
+ 'PASSED'))
+
+class OETestContextExecutor(object):
+ _context_class = OETestContext
+
+ name = 'core'
+ help = 'core test component example'
+ description = 'executes core test suite example'
+
+ default_cases = [os.path.join(os.path.abspath(os.path.dirname(__file__)),
+ 'cases/example')]
+ default_test_data = os.path.join(default_cases[0], 'data.json')
+
+ def register_commands(self, logger, subparsers):
+ self.parser = subparsers.add_parser(self.name, help=self.help,
+ description=self.description, group='components')
+
+ self.default_output_log = '%s-results-%s.log' % (self.name,
+ time.strftime("%Y%m%d%H%M%S"))
+ self.parser.add_argument('--output-log', action='store',
+ default=self.default_output_log,
+ help="results output log, default: %s" % self.default_output_log)
+
+ if self.default_test_data:
+ self.parser.add_argument('--test-data-file', action='store',
+ default=self.default_test_data,
+ help="data file to load, default: %s" % self.default_test_data)
+ else:
+ self.parser.add_argument('--test-data-file', action='store',
+ help="data file to load")
+
+ if self.default_cases:
+ self.parser.add_argument('CASES_PATHS', action='store',
+ default=self.default_cases, nargs='*',
+ help="paths to directories with test cases, default: %s"\
+ % self.default_cases)
+ else:
+ self.parser.add_argument('CASES_PATHS', action='store',
+ nargs='+', help="paths to directories with test cases")
+
+ self.parser.set_defaults(func=self.run)
+
+ def _setup_logger(self, logger, args):
+ formatter = logging.Formatter('%(asctime)s - ' + self.name + \
+ ' - %(levelname)s - %(message)s')
+ sh = logger.handlers[0]
+ sh.setFormatter(formatter)
+ fh = logging.FileHandler(args.output_log)
+ fh.setFormatter(formatter)
+ logger.addHandler(fh)
+
+ return logger
+
+ def _process_args(self, logger, args):
+ self.tc_kwargs = {}
+ self.tc_kwargs['init'] = {}
+ self.tc_kwargs['load'] = {}
+ self.tc_kwargs['run'] = {}
+
+ self.tc_kwargs['init']['logger'] = self._setup_logger(logger, args)
+ if args.test_data_file:
+ self.tc_kwargs['init']['td'] = json.load(
+ open(args.test_data_file, "r"))
+ else:
+ self.tc_kwargs['init']['td'] = {}
+
+ self.module_paths = args.CASES_PATHS
+
+ def run(self, logger, args):
+ self._process_args(logger, args)
+
+ self.tc = self._context_class(**self.tc_kwargs['init'])
+ self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
+ rc = self.tc.runTests(**self.tc_kwargs['run'])
+ self.tc.logSummary(rc, self.name)
+ self.tc.logDetails()
+
+ output_link = os.path.join(os.path.dirname(args.output_log),
+ "%s-results.log" % self.name)
+ if os.path.exists(output_link):
+ os.remove(output_link)
+ os.symlink(args.output_log, output_link)
+
+ return rc
+
+_executor_class = OETestContextExecutor
diff --git a/meta/lib/oeqa/core/decorator/__init__.py b/meta/lib/oeqa/core/decorator/__init__.py
new file mode 100644
index 0000000..855b6b9
--- /dev/null
+++ b/meta/lib/oeqa/core/decorator/__init__.py
@@ -0,0 +1,71 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from functools import wraps
+from abc import abstractmethod
+
+decoratorClasses = set()
+
+def registerDecorator(obj):
+ decoratorClasses.add(obj)
+ return obj
+
+class OETestDecorator(object):
+ case = None # Reference of OETestCase decorated
+ attrs = None # Attributes to be loaded by decorator implementation
+
+ def __init__(self, *args, **kwargs):
+ if not self.attrs:
+ return
+
+ for idx, attr in enumerate(self.attrs):
+ if attr in kwargs:
+ value = kwargs[attr]
+ else:
+ value = args[idx]
+ setattr(self, attr, value)
+
+ def __call__(self, func):
+ @wraps(func)
+ def wrapped_f(*args, **kwargs):
+ self.attrs = self.attrs # XXX: Enables OETestLoader discover
+ return func(*args, **kwargs)
+ return wrapped_f
+
+ # OETestLoader call it when is loading test cases.
+ # XXX: Most methods would change the registry for later
+ # processing; be aware that filtrate method needs to
+ # run later than bind, so there could be data (in the
+ # registry) of a cases that were filtered.
+ def bind(self, registry, case):
+ self.case = case
+ self.logger = case.tc.logger
+ self.case.decorators.append(self)
+
+ # OETestRunner call this method when tries to run
+ # the test case.
+ def setUpDecorator(self):
+ pass
+
+ # OETestRunner call it after a test method has been
+ # called even if the method raised an exception.
+ def tearDownDecorator(self):
+ pass
+
+class OETestDiscover(OETestDecorator):
+
+ # OETestLoader call it after discover test cases
+ # needs to return the cases to be run.
+ @staticmethod
+ def discover(registry):
+ return registry['cases']
+
+class OETestFilter(OETestDecorator):
+
+ # OETestLoader call it while loading the tests
+ # in loadTestsFromTestCase method, it needs to
+ # return a bool, True if needs to be filtered.
+ # This method must consume the filter used.
+ @abstractmethod
+ def filtrate(self, filters):
+ return False
diff --git a/meta/lib/oeqa/core/loader.py b/meta/lib/oeqa/core/loader.py
new file mode 100644
index 0000000..94f71ba
--- /dev/null
+++ b/meta/lib/oeqa/core/loader.py
@@ -0,0 +1,235 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import sys
+import unittest
+
+from oeqa.core.utils.path import findFile
+from oeqa.core.utils.test import getSuiteModules, getCaseID
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator import decoratorClasses, OETestDecorator, \
+ OETestFilter, OETestDiscover
+
+def _make_failed_test(classname, methodname, exception, suiteClass):
+ """
+ When loading tests unittest framework stores the exception in a new
+ class created for be displayed into run().
+
+ For our purposes will be better to raise the exception in loading
+ step instead of wait to run the test suite.
+ """
+ raise exception
+unittest.loader._make_failed_test = _make_failed_test
+
+def _find_duplicated_modules(suite, directory):
+ for module in getSuiteModules(suite):
+ path = findFile('%s.py' % module, directory)
+ if path:
+ raise ImportError("Duplicated %s module found in %s" % (module, path))
+
+class OETestLoader(unittest.TestLoader):
+ caseClass = OETestCase
+
+ kwargs_names = ['testMethodPrefix', 'sortTestMethodUsing', 'suiteClass',
+ '_top_level_dir']
+
+ def __init__(self, tc, module_paths, modules, tests, modules_required,
+ filters, *args, **kwargs):
+ self.tc = tc
+
+ self.modules = modules
+ self.tests = tests
+ self.modules_required = modules_required
+
+ self.filters = filters
+ self.decorator_filters = [d for d in decoratorClasses if \
+ issubclass(d, OETestFilter)]
+ self._validateFilters(self.filters, self.decorator_filters)
+ self.used_filters = [d for d in self.decorator_filters
+ for f in self.filters
+ if f in d.attrs]
+
+ if isinstance(module_paths, str):
+ module_paths = [module_paths]
+ elif not isinstance(module_paths, list):
+ raise TypeError('module_paths must be a str or a list of str')
+ self.module_paths = module_paths
+
+ for kwname in self.kwargs_names:
+ if kwname in kwargs:
+ setattr(self, kwname, kwargs[kwname])
+
+ self._patchCaseClass(self.caseClass)
+
+ def _patchCaseClass(self, testCaseClass):
+ # Adds custom attributes to the OETestCase class
+ setattr(testCaseClass, 'tc', self.tc)
+ setattr(testCaseClass, 'd', self.tc.d)
+ setattr(testCaseClass, 'logger', self.tc.logger)
+
+ def _validateFilters(self, filters, decorator_filters):
+ # Validate if filter isn't empty
+ for key,value in filters.items():
+ if not value:
+ raise TypeError("Filter %s specified is empty" % key)
+
+ # Validate unique attributes
+ attr_filters = [attr for clss in decorator_filters \
+ for attr in clss.attrs]
+ dup_attr = [attr for attr in attr_filters
+ if attr_filters.count(attr) > 1]
+ if dup_attr:
+ raise TypeError('Detected duplicated attribute(s) %s in filter'
+ ' decorators' % ' ,'.join(dup_attr))
+
+ # Validate if filter is supported
+ for f in filters:
+ if f not in attr_filters:
+ classes = ', '.join([d.__name__ for d in decorator_filters])
+ raise TypeError('Found "%s" filter but not declared in any of '
+ '%s decorators' % (f, classes))
+
+ def _registerTestCase(self, case):
+ case_id = case.id()
+ self.tc._registry['cases'][case_id] = case
+
+ def _handleTestCaseDecorators(self, case):
+ def _handle(obj):
+ if isinstance(obj, OETestDecorator):
+ if not obj.__class__ in decoratorClasses:
+ raise Exception("Decorator %s isn't registered" \
+ " in decoratorClasses." % obj.__name__)
+ obj.bind(self.tc._registry, case)
+
+ def _walk_closure(obj):
+ if hasattr(obj, '__closure__') and obj.__closure__:
+ for f in obj.__closure__:
+ obj = f.cell_contents
+ _handle(obj)
+ _walk_closure(obj)
+ method = getattr(case, case._testMethodName, None)
+ _walk_closure(method)
+
+ def _filterTest(self, case):
+ """
+ Returns True if test case must be filtered, False otherwise.
+ """
+ if self.filters:
+ filters = self.filters.copy()
+ case_decorators = [cd for cd in case.decorators
+ if cd.__class__ in self.used_filters]
+
+ # Iterate over case decorators to check if needs to be filtered.
+ for cd in case_decorators:
+ if cd.filtrate(filters):
+ return True
+
+ # Case is missing one or more decorators for all the filters
+ # being used, so filter test case.
+ if filters:
+ return True
+
+ return False
+
+ def _getTestCase(self, testCaseClass, tcName):
+ if not hasattr(testCaseClass, '__oeqa_loader'):
+ # In order to support data_vars validation
+ # monkey patch the default setUp/tearDown{Class} to use
+ # the ones provided by OETestCase
+ setattr(testCaseClass, 'setUpClassMethod',
+ getattr(testCaseClass, 'setUpClass'))
+ setattr(testCaseClass, 'tearDownClassMethod',
+ getattr(testCaseClass, 'tearDownClass'))
+ setattr(testCaseClass, 'setUpClass',
+ testCaseClass._oeSetUpClass)
+ setattr(testCaseClass, 'tearDownClass',
+ testCaseClass._oeTearDownClass)
+
+ # In order to support decorators initialization
+ # monkey patch the default setUp/tearDown to use
+ # a setUpDecorators/tearDownDecorators that methods
+ # will call setUp/tearDown original methods.
+ setattr(testCaseClass, 'setUpMethod',
+ getattr(testCaseClass, 'setUp'))
+ setattr(testCaseClass, 'tearDownMethod',
+ getattr(testCaseClass, 'tearDown'))
+ setattr(testCaseClass, 'setUp', testCaseClass._oeSetUp)
+ setattr(testCaseClass, 'tearDown', testCaseClass._oeTearDown)
+
+ setattr(testCaseClass, '__oeqa_loader', True)
+
+ case = testCaseClass(tcName)
+ setattr(case, 'decorators', [])
+
+ return case
+
+ def loadTestsFromTestCase(self, testCaseClass):
+ """
+ Returns a suite of all tests cases contained in testCaseClass.
+ """
+ if issubclass(testCaseClass, unittest.suite.TestSuite):
+ raise TypeError("Test cases should not be derived from TestSuite." \
+ " Maybe you meant to derive from TestCase?")
+ if not issubclass(testCaseClass, self.caseClass):
+ raise TypeError("Test cases need to be derived from %s" % \
+ caseClass.__name__)
+
+
+ testCaseNames = self.getTestCaseNames(testCaseClass)
+ if not testCaseNames and hasattr(testCaseClass, 'runTest'):
+ testCaseNames = ['runTest']
+
+ suite = []
+ for tcName in testCaseNames:
+ case = self._getTestCase(testCaseClass, tcName)
+ # Filer by case id
+ if not (self.tests and not 'all' in self.tests
+ and not getCaseID(case) in self.tests):
+ self._handleTestCaseDecorators(case)
+
+ # Filter by decorators
+ if not self._filterTest(case):
+ self._registerTestCase(case)
+ suite.append(case)
+
+ return self.suiteClass(suite)
+
+ def discover(self):
+ big_suite = self.suiteClass()
+ for path in self.module_paths:
+ _find_duplicated_modules(big_suite, path)
+ suite = super(OETestLoader, self).discover(path,
+ pattern='*.py', top_level_dir=path)
+ big_suite.addTests(suite)
+
+ cases = None
+ discover_classes = [clss for clss in decoratorClasses
+ if issubclass(clss, OETestDiscover)]
+ for clss in discover_classes:
+ cases = clss.discover(self.tc._registry)
+
+ return self.suiteClass(cases) if cases else big_suite
+
+ # XXX After Python 3.5, remove backward compatibility hacks for
+ # use_load_tests deprecation via *args and **kws. See issue 16662.
+ if sys.version_info >= (3,5):
+ def loadTestsFromModule(self, module, *args, pattern=None, **kws):
+ if not self.modules or "all" in self.modules or \
+ module.__name__ in self.modules:
+ return super(OETestLoader, self).loadTestsFromModule(
+ module, *args, pattern=pattern, **kws)
+ else:
+ return self.suiteClass()
+ else:
+ def loadTestsFromModule(self, module, use_load_tests=True):
+ """
+ Returns a suite of all tests cases contained in module.
+ """
+ if not self.modules or "all" in self.modules or \
+ module.__name__ in self.modules:
+ return super(OETestLoader, self).loadTestsFromModule(
+ module, use_load_tests)
+ else:
+ return self.suiteClass()
--
2.1.4
^ permalink raw reply related
* [PATCH 02/32] oeqa/core: Add utils module for OEQA framework
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
From: Mariano Lopez <mariano.lopez@linux.intel.com>
misc: Functions for transform object to other types.
path: Functions for path handling.
test: Functions for operations related to test cases and suites.
[YOCTO #10232]
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
meta/lib/oeqa/core/utils/__init__.py | 0
meta/lib/oeqa/core/utils/misc.py | 37 +++++++++++++++++++
meta/lib/oeqa/core/utils/path.py | 14 +++++++
meta/lib/oeqa/core/utils/test.py | 71 ++++++++++++++++++++++++++++++++++++
4 files changed, 122 insertions(+)
create mode 100644 meta/lib/oeqa/core/utils/__init__.py
create mode 100644 meta/lib/oeqa/core/utils/misc.py
create mode 100644 meta/lib/oeqa/core/utils/path.py
create mode 100644 meta/lib/oeqa/core/utils/test.py
diff --git a/meta/lib/oeqa/core/utils/__init__.py b/meta/lib/oeqa/core/utils/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/meta/lib/oeqa/core/utils/misc.py b/meta/lib/oeqa/core/utils/misc.py
new file mode 100644
index 0000000..6ae58ad
--- /dev/null
+++ b/meta/lib/oeqa/core/utils/misc.py
@@ -0,0 +1,37 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+def toList(obj, obj_type, obj_name="Object"):
+ if isinstance(obj, obj_type):
+ return [obj]
+ elif isinstance(obj, list):
+ return obj
+ else:
+ raise TypeError("%s must be %s or list" % (obj_name, obj_type))
+
+def toSet(obj, obj_type, obj_name="Object"):
+ if isinstance(obj, obj_type):
+ return {obj}
+ elif isinstance(obj, list):
+ return set(obj)
+ elif isinstance(obj, set):
+ return obj
+ else:
+ raise TypeError("%s must be %s or set" % (obj_name, obj_type))
+
+def strToList(obj, obj_name="Object"):
+ return toList(obj, str, obj_name)
+
+def strToSet(obj, obj_name="Object"):
+ return toSet(obj, str, obj_name)
+
+def intToList(obj, obj_name="Object"):
+ return toList(obj, int, obj_name)
+
+def dataStoteToDict(d, variables):
+ data = {}
+
+ for v in variables:
+ data[v] = d.getVar(v, True)
+
+ return data
diff --git a/meta/lib/oeqa/core/utils/path.py b/meta/lib/oeqa/core/utils/path.py
new file mode 100644
index 0000000..cb06523
--- /dev/null
+++ b/meta/lib/oeqa/core/utils/path.py
@@ -0,0 +1,14 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import sys
+
+def findFile(file_name, directory):
+ """
+ Search for a file in directory and returns its complete path.
+ """
+ for r, d, f in os.walk(directory):
+ if file_name in f:
+ return os.path.join(r, file_name)
+ return None
diff --git a/meta/lib/oeqa/core/utils/test.py b/meta/lib/oeqa/core/utils/test.py
new file mode 100644
index 0000000..820b997
--- /dev/null
+++ b/meta/lib/oeqa/core/utils/test.py
@@ -0,0 +1,71 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import unittest
+
+def getSuiteCases(suite):
+ """
+ Returns individual test from a test suite.
+ """
+ tests = []
+ for item in suite:
+ if isinstance(item, unittest.suite.TestSuite):
+ tests.extend(getSuiteCases(item))
+ elif isinstance(item, unittest.TestCase):
+ tests.append(item)
+ return tests
+
+def getSuiteModules(suite):
+ """
+ Returns modules in a test suite.
+ """
+ modules = set()
+ for test in getSuiteCases(suite):
+ modules.add(getCaseModule(test))
+ return modules
+
+def getSuiteCasesInfo(suite, func):
+ """
+ Returns test case info from suite. Info is fetched from func.
+ """
+ tests = []
+ for test in getSuiteCases(suite):
+ tests.append(func(test))
+ return tests
+
+def getSuiteCasesNames(suite):
+ """
+ Returns test case names from suite.
+ """
+ return getSuiteCasesInfo(suite, getCaseMethod)
+
+def getSuiteCasesIDs(suite):
+ """
+ Returns test case ids from suite.
+ """
+ return getSuiteCasesInfo(suite, getCaseID)
+
+def getCaseModule(test_case):
+ """
+ Returns test case module name.
+ """
+ return test_case.__module__
+
+def getCaseClass(test_case):
+ """
+ Returns test case class name.
+ """
+ return test_case.__class__.__name__
+
+def getCaseID(test_case):
+ """
+ Returns test case complete id.
+ """
+ return test_case.id()
+
+def getCaseMethod(test_case):
+ """
+ Returns test case method name.
+ """
+ return getCaseID(test_case).split('.')[-1]
--
2.1.4
^ permalink raw reply related
* [PATCH 01/32] oeqa/core: Add base OEQA framework
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
In-Reply-To: <cover.1481059445.git.anibal.limon@linux.intel.com>
case: Defines OETestCase base class that provides custom
methods/attrs defined by the framework.
Every OETestCase instance contains a reference to the test
data (d), the test context (tc) and the logger.
Also implements _oe{SetUp,TearDown}Class for make special
handling of OEQA decorators and validations.
runner: Defines OETestRunner/OETestResult with support for RAW
and XML result logs.
exception: Custom exceptions related to the OEQA framework based
on class OEQAException.
[YOCTO #10230]
[YOCTO #10233]
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
meta/lib/oeqa/core/__init__.py | 0
meta/lib/oeqa/core/case.py | 46 +++++++++++++++++++++++++
meta/lib/oeqa/core/exception.py | 14 ++++++++
meta/lib/oeqa/core/runner.py | 76 +++++++++++++++++++++++++++++++++++++++++
4 files changed, 136 insertions(+)
create mode 100644 meta/lib/oeqa/core/__init__.py
create mode 100644 meta/lib/oeqa/core/case.py
create mode 100644 meta/lib/oeqa/core/exception.py
create mode 100644 meta/lib/oeqa/core/runner.py
diff --git a/meta/lib/oeqa/core/__init__.py b/meta/lib/oeqa/core/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/meta/lib/oeqa/core/case.py b/meta/lib/oeqa/core/case.py
new file mode 100644
index 0000000..a47cb19
--- /dev/null
+++ b/meta/lib/oeqa/core/case.py
@@ -0,0 +1,46 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import unittest
+
+from oeqa.core.exception import OEQAMissingVariable
+
+def _validate_data_vars(d, data_vars, type_msg):
+ if data_vars:
+ for v in data_vars:
+ if not v in d:
+ raise OEQAMissingVariable("Test %s need %s variable but"\
+ " isn't into d" % (type_msg, v))
+
+class OETestCase(unittest.TestCase):
+ # TestContext and Logger instance set by OETestLoader.
+ tc = None
+ logger = None
+
+ # d has all the variables needed by the test cases
+ # is the same across all the test cases.
+ d = None
+
+ # data_vars has the variables needed by a test class
+ # or test case instance, if some var isn't into d a
+ # OEMissingVariable exception is raised
+ data_vars = None
+
+ @classmethod
+ def _oeSetUpClass(clss):
+ _validate_data_vars(clss.d, clss.data_vars, "class")
+ clss.setUpClassMethod()
+
+ @classmethod
+ def _oeTearDownClass(clss):
+ clss.tearDownClassMethod()
+
+ def _oeSetUp(self):
+ for d in self.decorators:
+ d.setUpDecorator()
+ self.setUpMethod()
+
+ def _oeTearDown(self):
+ for d in self.decorators:
+ d.tearDownDecorator()
+ self.tearDownMethod()
diff --git a/meta/lib/oeqa/core/exception.py b/meta/lib/oeqa/core/exception.py
new file mode 100644
index 0000000..2dfd840
--- /dev/null
+++ b/meta/lib/oeqa/core/exception.py
@@ -0,0 +1,14 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+class OEQAException(Exception):
+ pass
+
+class OEQATimeoutError(OEQAException):
+ pass
+
+class OEQAMissingVariable(OEQAException):
+ pass
+
+class OEQADependency(OEQAException):
+ pass
diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
new file mode 100644
index 0000000..8f5af57
--- /dev/null
+++ b/meta/lib/oeqa/core/runner.py
@@ -0,0 +1,76 @@
+# Copyright (C) 2016 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import time
+import unittest
+import logging
+
+xmlEnabled = False
+try:
+ import xmlrunner
+ from xmlrunner.result import _XMLTestResult as _TestResult
+ from xmlrunner.runner import XMLTestRunner as _TestRunner
+ xmlEnabled = True
+except ImportError:
+ # use the base runner instead
+ from unittest import TextTestResult as _TestResult
+ from unittest import TextTestRunner as _TestRunner
+
+class OEStreamLogger(object):
+ def __init__(self, logger):
+ self.logger = logger
+ self.buffer = ""
+
+ def write(self, msg):
+ if msg[-1] != '\n':
+ self.buffer += msg
+ else:
+ self.logger.log(logging.INFO, self.buffer.rstrip("\n"))
+ self.buffer = ""
+
+ def flush(self):
+ for handler in self.logger.handlers:
+ handler.flush()
+
+class OETestResult(_TestResult):
+ def __init__(self, tc, *args, **kwargs):
+ super(OETestResult, self).__init__(*args, **kwargs)
+
+ self.tc = tc
+
+ self.tc._results['failures'] = self.failures
+ self.tc._results['errors'] = self.errors
+ self.tc._results['skipped'] = self.skipped
+ self.tc._results['expectedFailures'] = self.expectedFailures
+
+ def startTest(self, test):
+ super(OETestResult, self).startTest(test)
+
+class OETestRunner(_TestRunner):
+ def __init__(self, tc, *args, **kwargs):
+ if xmlEnabled:
+ if not kwargs.get('output'):
+ kwargs['output'] = os.path.join(os.getcwd(),
+ 'TestResults_%s' % time.strftime("%Y%m%d%H%M%S"))
+
+ super(OETestRunner, self).__init__(*args, **kwargs)
+ self.tc = tc
+ self.resultclass = OETestResult
+
+ # XXX: The unittest-xml-reporting package defines _make_result method instead
+ # of _makeResult standard on unittest.
+ if xmlEnabled:
+ def _make_result(self):
+ """
+ Creates a TestResult object which will be used to store
+ information about the executed tests.
+ """
+ # override in subclasses if necessary.
+ return self.resultclass(self.tc,
+ self.stream, self.descriptions, self.verbosity, self.elapsed_times
+ )
+ else:
+ def _makeResult(self):
+ return self.resultclass(self.tc, self.stream, self.descriptions,
+ self.verbosity)
--
2.1.4
^ permalink raw reply related
* [PATCH 00/32] OEQA Framework Refactor & Improvements
From: Aníbal Limón @ 2016-12-06 21:43 UTC (permalink / raw)
To: openembedded-core
This patchset is related to OEQA Framework for details read the RFC send to the
Openembedded architecture ML.
The following changes since commit 9e63f81c78e284c9b325fe04a1b59e61c7ad8a1a:
bitbake: ast: remove BBVERSIONS support (2016-11-30 15:48:10 +0000)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib alimon/oeqa_sdk_migration
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=alimon/oeqa_sdk_migration
Aníbal Limón (28):
oeqa/core: Add base OEQA framework
oeqa/core: Add loader, context and decorator modules
oeqa/core/decorator: Add support for OETestDepends
oeqa/core/decorator: Add support for OETestDataDepends and
skipIfDataVar
scripts/oe-test: Add new oe-test script
oeqa/core/context: Add support of OETestContextExecutor
oeqa/core/cases: Add example test cases
oeqa/core: Add README
oe/data: Add export2json function
classes/rootfs-postcommands: Add write_image_test_data
classes/populate_sdk_base: Add write_sdk_test_data to postprocess
oeqa/core: Change name of d to td
oeqa/utils/path: Add remove_safe function
oeqa: Move common files to oeqa/files instead of runtime only
oeqa/sdk: Move test cases inside cases directory
oeqa/{runtime,sdk}/files: Move testsdkmakefile from runtime to sdk
module
oeqa/sdk: Add case and context modules for the SDK component
classes/testsdk: Migrates testsdk.bbclass to use new OESDKTestContext
oeqa/utils: Move targetbuild to buildproject module
oeqa/utils: {Target,SDK,}BuildProject remove dependency of bb
oeqa/sdk/cases: Migrate tests to the new OEQA framework
classes/testsdk: Remove the need of TEST_LOG_DIR variable
oeqa/sdkext: Move test cases inside cases directory
oeqa/sdkext: Adds case and context modules.
classes/testsdk: Migrate to use the new OESDKExtTestContext
oeqa/sdkext/cases: Migrate test case to new OEQA framework
oeqa/runtime: Fix TargetBuildProject instances
oeqa: Fix files handling on runtime tests.
Mariano Lopez (4):
oeqa/core: Add utils module for OEQA framework
oeqa/core/decorator: Add support for OETestID and OETestTag
oeqa/core/decorator: Add support for OETimeout decorator
oeqa/core: Add tests for the OEQA framework
meta/classes/populate_sdk_base.bbclass | 9 +-
meta/classes/rootfs-postcommands.bbclass | 18 ++
meta/classes/testexport.bbclass | 5 +
meta/classes/testimage.bbclass | 4 +
meta/classes/testsdk.bbclass | 162 ++++++++------
meta/lib/oe/data.py | 28 +++
meta/lib/oeqa/core/README | 38 ++++
meta/lib/oeqa/core/__init__.py | 0
meta/lib/oeqa/core/case.py | 46 ++++
meta/lib/oeqa/core/cases/__init__.py | 0
meta/lib/oeqa/core/cases/example/data.json | 1 +
meta/lib/oeqa/core/cases/example/test_basic.py | 20 ++
meta/lib/oeqa/core/context.py | 225 ++++++++++++++++++++
meta/lib/oeqa/core/decorator/__init__.py | 71 +++++++
meta/lib/oeqa/core/decorator/data.py | 36 ++++
meta/lib/oeqa/core/decorator/depends.py | 94 +++++++++
meta/lib/oeqa/core/decorator/oeid.py | 23 ++
meta/lib/oeqa/core/decorator/oetag.py | 24 +++
meta/lib/oeqa/core/decorator/oetimeout.py | 25 +++
meta/lib/oeqa/core/exception.py | 14 ++
meta/lib/oeqa/core/loader.py | 235 +++++++++++++++++++++
meta/lib/oeqa/core/runner.py | 76 +++++++
meta/lib/oeqa/core/tests/__init__.py | 0
meta/lib/oeqa/core/tests/cases/data.py | 20 ++
meta/lib/oeqa/core/tests/cases/depends.py | 38 ++++
.../oeqa/core/tests/cases/loader/invalid/oeid.py | 15 ++
.../oeqa/core/tests/cases/loader/valid/another.py | 9 +
meta/lib/oeqa/core/tests/cases/oeid.py | 18 ++
meta/lib/oeqa/core/tests/cases/oetag.py | 18 ++
meta/lib/oeqa/core/tests/cases/timeout.py | 18 ++
meta/lib/oeqa/core/tests/common.py | 35 +++
meta/lib/oeqa/core/tests/test_data.py | 51 +++++
meta/lib/oeqa/core/tests/test_decorators.py | 135 ++++++++++++
meta/lib/oeqa/core/tests/test_loader.py | 86 ++++++++
meta/lib/oeqa/core/tests/test_runner.py | 38 ++++
meta/lib/oeqa/core/utils/__init__.py | 0
meta/lib/oeqa/core/utils/misc.py | 37 ++++
meta/lib/oeqa/core/utils/path.py | 19 ++
meta/lib/oeqa/core/utils/test.py | 71 +++++++
meta/lib/oeqa/{runtime => }/files/test.c | 0
meta/lib/oeqa/{runtime => }/files/test.cpp | 0
meta/lib/oeqa/{runtime => }/files/test.pl | 0
meta/lib/oeqa/{runtime => }/files/test.py | 0
meta/lib/oeqa/oetest.py | 92 +-------
meta/lib/oeqa/runtime/buildcvs.py | 9 +-
meta/lib/oeqa/runtime/buildgalculator.py | 8 +-
meta/lib/oeqa/runtime/buildiptables.py | 8 +-
meta/lib/oeqa/runtime/gcc.py | 4 +-
meta/lib/oeqa/runtime/perl.py | 2 +-
meta/lib/oeqa/runtime/python.py | 2 +-
meta/lib/oeqa/runtime/utils/__init__.py | 0
meta/lib/oeqa/runtime/utils/targetbuildproject.py | 33 +++
meta/lib/oeqa/sdk/__init__.py | 3 -
meta/lib/oeqa/sdk/case.py | 12 ++
meta/lib/oeqa/sdk/{ => cases}/buildcvs.py | 15 +-
meta/lib/oeqa/sdk/{ => cases}/buildgalculator.py | 28 ++-
meta/lib/oeqa/sdk/{ => cases}/buildiptables.py | 16 +-
meta/lib/oeqa/sdk/cases/gcc.py | 43 ++++
meta/lib/oeqa/sdk/cases/perl.py | 27 +++
meta/lib/oeqa/sdk/{ => cases}/python.py | 25 ++-
meta/lib/oeqa/sdk/context.py | 133 ++++++++++++
.../oeqa/{runtime => sdk}/files/testsdkmakefile | 0
meta/lib/oeqa/sdk/gcc.py | 36 ----
meta/lib/oeqa/sdk/perl.py | 28 ---
meta/lib/oeqa/sdk/utils/__init__.py | 0
meta/lib/oeqa/sdk/utils/sdkbuildproject.py | 45 ++++
meta/lib/oeqa/sdkext/__init__.py | 3 -
meta/lib/oeqa/sdkext/case.py | 21 ++
meta/lib/oeqa/sdkext/{ => cases}/devtool.py | 49 +++--
meta/lib/oeqa/sdkext/{ => cases}/sdk_update.py | 14 +-
meta/lib/oeqa/sdkext/context.py | 21 ++
meta/lib/oeqa/utils/buildproject.py | 52 +++++
meta/lib/oeqa/utils/targetbuild.py | 135 ------------
scripts/oe-test | 98 +++++++++
74 files changed, 2282 insertions(+), 442 deletions(-)
create mode 100644 meta/lib/oeqa/core/README
create mode 100644 meta/lib/oeqa/core/__init__.py
create mode 100644 meta/lib/oeqa/core/case.py
create mode 100644 meta/lib/oeqa/core/cases/__init__.py
create mode 100644 meta/lib/oeqa/core/cases/example/data.json
create mode 100644 meta/lib/oeqa/core/cases/example/test_basic.py
create mode 100644 meta/lib/oeqa/core/context.py
create mode 100644 meta/lib/oeqa/core/decorator/__init__.py
create mode 100644 meta/lib/oeqa/core/decorator/data.py
create mode 100644 meta/lib/oeqa/core/decorator/depends.py
create mode 100644 meta/lib/oeqa/core/decorator/oeid.py
create mode 100644 meta/lib/oeqa/core/decorator/oetag.py
create mode 100644 meta/lib/oeqa/core/decorator/oetimeout.py
create mode 100644 meta/lib/oeqa/core/exception.py
create mode 100644 meta/lib/oeqa/core/loader.py
create mode 100644 meta/lib/oeqa/core/runner.py
create mode 100644 meta/lib/oeqa/core/tests/__init__.py
create mode 100644 meta/lib/oeqa/core/tests/cases/data.py
create mode 100644 meta/lib/oeqa/core/tests/cases/depends.py
create mode 100644 meta/lib/oeqa/core/tests/cases/loader/invalid/oeid.py
create mode 100644 meta/lib/oeqa/core/tests/cases/loader/valid/another.py
create mode 100644 meta/lib/oeqa/core/tests/cases/oeid.py
create mode 100644 meta/lib/oeqa/core/tests/cases/oetag.py
create mode 100644 meta/lib/oeqa/core/tests/cases/timeout.py
create mode 100644 meta/lib/oeqa/core/tests/common.py
create mode 100755 meta/lib/oeqa/core/tests/test_data.py
create mode 100755 meta/lib/oeqa/core/tests/test_decorators.py
create mode 100755 meta/lib/oeqa/core/tests/test_loader.py
create mode 100755 meta/lib/oeqa/core/tests/test_runner.py
create mode 100644 meta/lib/oeqa/core/utils/__init__.py
create mode 100644 meta/lib/oeqa/core/utils/misc.py
create mode 100644 meta/lib/oeqa/core/utils/path.py
create mode 100644 meta/lib/oeqa/core/utils/test.py
rename meta/lib/oeqa/{runtime => }/files/test.c (100%)
rename meta/lib/oeqa/{runtime => }/files/test.cpp (100%)
rename meta/lib/oeqa/{runtime => }/files/test.pl (100%)
rename meta/lib/oeqa/{runtime => }/files/test.py (100%)
create mode 100644 meta/lib/oeqa/runtime/utils/__init__.py
create mode 100644 meta/lib/oeqa/runtime/utils/targetbuildproject.py
create mode 100644 meta/lib/oeqa/sdk/case.py
rename meta/lib/oeqa/sdk/{ => cases}/buildcvs.py (59%)
rename meta/lib/oeqa/sdk/{ => cases}/buildgalculator.py (45%)
rename meta/lib/oeqa/sdk/{ => cases}/buildiptables.py (58%)
create mode 100644 meta/lib/oeqa/sdk/cases/gcc.py
create mode 100644 meta/lib/oeqa/sdk/cases/perl.py
rename meta/lib/oeqa/sdk/{ => cases}/python.py (46%)
create mode 100644 meta/lib/oeqa/sdk/context.py
rename meta/lib/oeqa/{runtime => sdk}/files/testsdkmakefile (100%)
delete mode 100644 meta/lib/oeqa/sdk/gcc.py
delete mode 100644 meta/lib/oeqa/sdk/perl.py
create mode 100644 meta/lib/oeqa/sdk/utils/__init__.py
create mode 100644 meta/lib/oeqa/sdk/utils/sdkbuildproject.py
create mode 100644 meta/lib/oeqa/sdkext/case.py
rename meta/lib/oeqa/sdkext/{ => cases}/devtool.py (73%)
rename meta/lib/oeqa/sdkext/{ => cases}/sdk_update.py (69%)
create mode 100644 meta/lib/oeqa/sdkext/context.py
create mode 100644 meta/lib/oeqa/utils/buildproject.py
delete mode 100644 meta/lib/oeqa/utils/targetbuild.py
create mode 100755 scripts/oe-test
--
2.1.4
^ permalink raw reply
* Re: [PATCH] run-postinsts: Print message before running deferred postinst scripts
From: Haris Okanovic @ 2016-12-06 19:01 UTC (permalink / raw)
To: Burton, Ross; +Cc: OE-core
In-Reply-To: <CAJTo0La+W7qJikWqjHh6f90M_Cr_v2EUwegE+CGRLYCR5D6K8A@mail.gmail.com>
On 12/05/2016 04:51 PM, Burton, Ross wrote:
>
> On 5 December 2016 at 21:48, Haris Okanovic <haris.okanovic@ni.com
> <mailto:haris.okanovic@ni.com>> wrote:
>
> Opkg can defer running postinst scripts to first boot, which can take
> a while on some systems. The output of `opkg configure` (or whatever pm
> is used) is redirected to a file when logging is enabled
> (I.e. $POSTINST_LOGGING == 1), making the machine appear hung during
> this process. This change simply prints a wait message on the console
> to inform the user of this potentially long and silent operation so
> that they do not mistakenly reboot their machine.
>
>
> This isn't opkg specific, all backends can do it.
I wrote a more generic commit message in V2.
>
>
> Why not simply `tee` the output instead?
> Tee might be provided by BusyBox in some distros, which may need to run
> update-alternatives in the very postinst scripts being executed by this
> process. It's therefore not safe to assume Tee (or any other packaged
> util) is available until the configure process finishes.
>
>
> Are the alternatives not configured at rootfs time, so it should be fine
> to run tee? (as if tee isn't safe, then neither is sed).
`tee` wasn't configured in Fido builds of NI Linux RT -- I.e.
Busybox.postinst got deferred. I'm not sure if that's the OE default or
the result of some distro-specific tweaks we made, but I decided not to
rely on it for that reason. It's possible tee might work on some
configurations.
I'm not sure why sed is significant. The only non-builtins in
run-postinsts are update-rc.d, rm, and $pm.
>
> Ross
^ permalink raw reply
* Re: [PATCH 3/4] kern-tools: fix processing for no branch meta-data
From: Patrick Ohly @ 2016-12-06 21:14 UTC (permalink / raw)
To: Bruce Ashfield, Mikko Ylinen; +Cc: openembedded-core
In-Reply-To: <48f1e3e16ca28e1194d6024689d000cb6ffc303c.1480711764.git.bruce.ashfield@windriver.com>
On Fri, 2016-12-02 at 16:09 -0500, Bruce Ashfield wrote:
> Lernel meta-data that has patches, but no branches, can trigger an
> error due to no branch specific patch queue.
>
> This error then cascades to more issues since the tools are using
> a named file in /tmp to store and display error messages to the
> user.
>
> We fix both issues though the following kern tools tweaks:
>
> commit bd9e1d6c9b0a34ff3e19a06999aaf57ffadfd04c
> Author: Bruce Ashfield <bruce.ashfield@windriver.com>
> Date: Fri Dec 2 13:09:40 2016 -0500
>
> scc: use mktemp for consolidated output capture
>
> To provide useful error messages the tools dump pre-processed
> files and messages to a temporary file. If multiple users are
> doing builds, this means they either race, or can have permissions
> issues.
>
> By creating the temporary file via mktemp, we avoid both issues.
> (We also make sure to clean these up on exit, or /tmp will get
> polluted quickly).
>
> commit a287da4bfe0b4acb8f2b0627bd8e7abd1a1dde26
> Author: Bruce Ashfield <bruce.ashfield@windriver.com>
> Date: Fri Dec 2 13:08:08 2016 -0500
>
> patch: do not assume a branch specific patch queue is needed
>
> When processing input files per-branch and global patch queues are
> generated. If the meta-data has not created any branches in the
> repo, no branch specific queue is required.
>
> The tools assumed that one is always valid, and hence would throw a
> non-zero exit code and stop processing.
>
> By testing for a named per-branch queue, we avoid this issue.
Ostro OS runs into the problem while trying to use current OE-core
master:
.../patch.cmd: line 29: : No such file or directory
| ERROR: Function failed: do_kernel_metadata (log file is located ...)
This commit here fixed it for me. I see that it is already in Ross' mut2
branch, so hopefully that'll land in master soon.
--
Best Regards, Patrick Ohly
The content of this message is my personal opinion only and although
I am an employee of Intel, the statements I make here in no way
represent Intel's position on the issue, nor am I authorized to speak
on behalf of Intel on this matter.
^ permalink raw reply
* Re: [PATCH 28/33] dpkg: update packages and files to match Debian more closely
From: Burton, Ross @ 2016-12-06 20:13 UTC (permalink / raw)
To: Andreas Oberritter; +Cc: OE-core
In-Reply-To: <1481024991-5882-29-git-send-email-obi@opendreambox.org>
[-- Attachment #1: Type: text/plain, Size: 737 bytes --]
On 6 December 2016 at 11:49, Andreas Oberritter <obi@opendreambox.org>
wrote:
> +RRECOMMENDS_dpkg-perl = "gnupg gpgv"
>
With plain master this causes a build failure:
ERROR: Multiple versions of gnupg are due to be built
(/home/ross/Yocto/poky/meta/recipes-support/gnupg/gnupg_2.1.14.bb
/home/ross/Yocto/poky/meta/recipes-support/gnupg/gnupg_1.4.7.bb). Only one
version of a given PN should be built in any given build. You likely need
to set PREFERRED_VERSION_gnupg to select the correct version or don't
depend on multiple versions.
As gnupg 1.4.7 provides gpgv, but 2.1.14 doesn't.
No idea where 2.1.14 should, what gpgv is, or what the solution is, as it's
late and I'm going to crash on the sofa. :)
Ross
[-- Attachment #2: Type: text/html, Size: 1541 bytes --]
^ permalink raw reply
* Re: [PATCH 02/33] dpkg: update-alternatives-dpkg should conflict with other providers
From: Burton, Ross @ 2016-12-06 19:52 UTC (permalink / raw)
To: Andreas Oberritter; +Cc: OE-core
In-Reply-To: <1481024991-5882-3-git-send-email-obi@opendreambox.org>
[-- Attachment #1: Type: text/plain, Size: 1538 bytes --]
This needs more than just RCONFLICTS:
ERROR: dpkg-native-1.18.7-r0 do_populate_sysroot: The recipe dpkg-native is
trying to install files into a shared area when those files already exist.
Those files and their manifest location are:
/data/poky-master/tmp-glibc/sysroots/x86_64-linux/usr/bin/update-alternatives
I have both package_deb and package_rpm enabled, and have a build from
master before trying your branch.
Ross
On 6 December 2016 at 11:49, Andreas Oberritter <obi@opendreambox.org>
wrote:
> Signed-off-by: Andreas Oberritter <obi@opendreambox.org>
> ---
> meta/recipes-devtools/dpkg/dpkg.inc | 1 +
> 1 file changed, 1 insertion(+)
>
> diff --git a/meta/recipes-devtools/dpkg/dpkg.inc
> b/meta/recipes-devtools/dpkg/dpkg.inc
> index ec0117b..f7d9e77 100644
> --- a/meta/recipes-devtools/dpkg/dpkg.inc
> +++ b/meta/recipes-devtools/dpkg/dpkg.inc
> @@ -63,6 +63,7 @@ do_install_append () {
> PACKAGES =+ "update-alternatives-dpkg"
> FILES_update-alternatives-dpkg = "${bindir}/update-alternatives
> ${localstatedir}/lib/dpkg/alternatives ${sysconfdir}/alternatives"
> RPROVIDES_update-alternatives-dpkg = "update-alternatives"
> +RCONFLICTS_update-alternatives-dpkg = "update-alternatives"
>
> PACKAGES += "${PN}-perl"
> FILES_${PN}-perl = "${libdir}/perl"
> --
> 2.7.4
>
> --
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/openembedded-core
>
[-- Attachment #2: Type: text/html, Size: 2363 bytes --]
^ permalink raw reply
* [PATCH v2] run-postinsts: Print message before running deferred postinst scripts
From: Haris Okanovic @ 2016-12-06 19:51 UTC (permalink / raw)
To: openembedded-core; +Cc: Haris Okanovic
In-Reply-To: <CAJTo0La+W7qJikWqjHh6f90M_Cr_v2EUwegE+CGRLYCR5D6K8A@mail.gmail.com>
Package managers can defer running postinst scripts to first boot, which
can take a while on some systems. E.g. The output of `opkg configure`
(or whatever pm is used) is redirected to a file when logging is enabled
($POSTINST_LOGGING == 1), making the machine appear hung during this
process. This change simply prints a wait message on the console to
inform the user of this potentially long and silent operation so they do
not mistakenly reboot the machine.
Why not simply `tee` the output instead?
Tee might be provided by BusyBox in some distros, which may need to run
update-alternatives in the very postinst scripts being executed by this
process. It's therefore not safe to assume Tee (or any other packaged
util) is available until the configure process finishes.
Signed-off-by: Haris Okanovic <haris.okanovic@ni.com>
---
meta/recipes-devtools/run-postinsts/run-postinsts/run-postinsts | 3 +++
1 file changed, 3 insertions(+)
diff --git a/meta/recipes-devtools/run-postinsts/run-postinsts/run-postinsts b/meta/recipes-devtools/run-postinsts/run-postinsts/run-postinsts
index 04ba394..660ddc2 100755
--- a/meta/recipes-devtools/run-postinsts/run-postinsts/run-postinsts
+++ b/meta/recipes-devtools/run-postinsts/run-postinsts/run-postinsts
@@ -46,6 +46,9 @@ if [ -z "$pi_dir" ]; then
exit 0
fi
+echo "Configuring packages on first boot...."
+echo " (This may take several minutes. Please do not power off the machine.)"
+
[ -e #SYSCONFDIR#/default/postinst ] && . #SYSCONFDIR#/default/postinst
if [ "$POSTINST_LOGGING" = "1" ]; then
--
2.10.1
^ permalink raw reply related
* Re: [PATCH 0/4] kernel-yocto: consolidated pull request
From: Trevor Woerner @ 2016-12-06 19:32 UTC (permalink / raw)
To: Bruce Ashfield; +Cc: openembedded-core
In-Reply-To: <cover.1480711764.git.bruce.ashfield@windriver.com>
On Fri 2016-12-02 @ 04:09:21 PM, Bruce Ashfield wrote:
> This pull request is mainly to fix a couple of bugs that were reported
> on the mailing list recently, but it also includes some kernel version
> updates that I *think* I sent previously.
Thanks Bruce, I've tested these and they look good.
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox