[PATCH 1/2] oeqa/{core, selftest}: Add support to validate if a specified test case isn't found

[PATCH 1/2] oeqa/{core, selftest}: Add support to validate if a specified test case isn't found

[Aníbal Limón]

If some test module/case is specified to run and isn't found the OEQA
framework didn't notice it, so complete the implementation using
modules_required and validate for the test case prescense.

Raise an exception when the test module/case required isn't found.

[YOCTO #11645]

Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
 meta/lib/oeqa/core/context.py     |  9 +++++++--
 meta/lib/oeqa/core/exception.py   |  3 +++
 meta/lib/oeqa/core/loader.py      | 26 ++++++++++++++++++++++++++
 meta/lib/oeqa/selftest/context.py |  8 ++++++--
 4 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
index 2d543ffa31a..422e2899921 100644
--- a/meta/lib/oeqa/core/context.py
+++ b/meta/lib/oeqa/core/context.py
@@ -10,7 +10,7 @@ import collections
 
 from oeqa.core.loader import OETestLoader
 from oeqa.core.runner import OETestRunner
-from oeqa.core.exception import OEQAMissingManifest
+from oeqa.core.exception import OEQAMissingManifest, OEQATestNotFound
 
 class OETestContext(object):
     loaderClass = OETestLoader
@@ -139,6 +139,7 @@ class OETestContextExecutor(object):
 
         if args.run_tests:
             self.tc_kwargs['load']['modules'] = args.run_tests
+            self.tc_kwargs['load']['modules_required'] = args.run_tests
         else:
             self.tc_kwargs['load']['modules'] = []
 
@@ -151,7 +152,11 @@ class OETestContextExecutor(object):
         self._process_args(logger, args)
 
         self.tc = self._context_class(**self.tc_kwargs['init'])
-        self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
+        try:
+            self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
+        except OEQATestNotFound as ex:
+            logger.error(ex)
+            sys.exit(1)
 
         if args.list_tests:
             rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['run'])
diff --git a/meta/lib/oeqa/core/exception.py b/meta/lib/oeqa/core/exception.py
index a07961adc31..732f2efdeb6 100644
--- a/meta/lib/oeqa/core/exception.py
+++ b/meta/lib/oeqa/core/exception.py
@@ -18,3 +18,6 @@ class OEQAMissingManifest(OEQAException):
 
 class OEQAPreRun(OEQAException):
     pass
+
+class OEQATestNotFound(OEQAException):
+    pass
diff --git a/meta/lib/oeqa/core/loader.py b/meta/lib/oeqa/core/loader.py
index e4c218b57f0..332086a13d7 100644
--- a/meta/lib/oeqa/core/loader.py
+++ b/meta/lib/oeqa/core/loader.py
@@ -9,6 +9,7 @@ import inspect
 from oeqa.core.utils.path import findFile
 from oeqa.core.utils.test import getSuiteModules, getCaseID
 
+from oeqa.core.exception import OEQATestNotFound
 from oeqa.core.case import OETestCase
 from oeqa.core.decorator import decoratorClasses, OETestDecorator, \
         OETestFilter, OETestDiscover
@@ -277,6 +278,28 @@ class OETestLoader(unittest.TestLoader):
 
         return self.suiteClass(suite)
 
+    def _required_modules_validation(self):
+        """
+            Search in Test context registry if a required
+            test is found, raise an exception when not found.
+        """
+
+        for module in self.modules_required:
+            found = False
+
+            # The module name is splitted to only compare the
+            # first part of a test case id.
+            comp_len = len(module.split('.'))
+            for case in self.tc._registry['cases']:
+                case_comp = '.'.join(case.split('.')[0:comp_len])
+                if module == case_comp:
+                    found = True
+                    break
+
+            if not found:
+                raise OEQATestNotFound("Not found %s in loaded test cases" % \
+                        module)
+
     def discover(self):
         big_suite = self.suiteClass()
         for path in self.module_paths:
@@ -291,6 +314,9 @@ class OETestLoader(unittest.TestLoader):
         for clss in discover_classes:
             cases = clss.discover(self.tc._registry)
 
+        if self.modules_required:
+            self._required_modules_validation()
+
         return self.suiteClass(cases) if cases else big_suite
 
     def _filterModule(self, module):
diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py
index 4575a0537fb..990c761f298 100644
--- a/meta/lib/oeqa/selftest/context.py
+++ b/meta/lib/oeqa/selftest/context.py
@@ -13,7 +13,7 @@ from random import choice
 import oeqa
 
 from oeqa.core.context import OETestContext, OETestContextExecutor
-from oeqa.core.exception import OEQAPreRun
+from oeqa.core.exception import OEQAPreRun, OEQATestNotFound
 
 from oeqa.utils.commands import runCmd, get_bb_vars, get_test_layer
 
@@ -196,7 +196,11 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
                 self.tc_kwargs['init']['td']['BBPATH'].split(':'))
 
         self.tc = self._context_class(**self.tc_kwargs['init'])
-        self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
+        try:
+            self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
+        except OEQATestNotFound as ex:
+            logger.error(ex)
+            sys.exit(1)
 
         if args.list_tests:
             rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['run'])
-- 
2.11.0

[PATCH 2/2] oeqa/core/runner: OEStreamLogger don't buffer test execution writes

[Aníbal Limón]

Since OEQA framework uses Python logging functionality to report test
results there is a class that wraps PyUnit writes into logging commands
(OEStreamLogger), so don't buffer the actual test execution to have
insight of what is currently executing.

This fix will change a little the test output format adding an '\n'
previous the test result, for example:

From:

test_nonmatching_checksum (lic_checksum.LicenseTests) ... ok

To:

test_nonmatching_checksum (lic_checksum.LicenseTests)
 ... ok

This is because the new line added by the PyUnit StreamLogger because
currently we don't have a manner to identify when a test execution
starts at report level (write msg).

[YOCTO #11827]

Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
 meta/lib/oeqa/core/runner.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
index 8a55c24c788..f6539e60b6b 100644
--- a/meta/lib/oeqa/core/runner.py
+++ b/meta/lib/oeqa/core/runner.py
@@ -25,10 +25,14 @@ class OEStreamLogger(object):
 
     def write(self, msg):
         if len(msg) > 1 and msg[0] != '\n':
-            self.buffer += msg
-        else:
-            self.logger.log(logging.INFO, self.buffer.rstrip("\n"))
-            self.buffer = ""
+            if '...' in msg:
+                self.buffer += msg
+            elif self.buffer:
+                self.buffer += msg
+                self.logger.log(logging.INFO, self.buffer)
+                self.buffer = ""
+            else:
+                self.logger.log(logging.INFO, msg)
 
     def flush(self):
         for handler in self.logger.handlers:
-- 
2.11.0