* [PATCH 1/3] oeqa/oetest.py: use bb logging
2015-04-09 9:24 [PATCH 0/3] testimage improvements Patrick Ohly
@ 2015-04-09 9:24 ` Patrick Ohly
2015-04-09 9:24 ` [PATCH 2/3] oeqa/runtime/systemd.py: skip instead of failing without avahi Patrick Ohly
2015-04-09 9:24 ` [PATCH 3/3] testimage: sort modules based on dependencies Patrick Ohly
2 siblings, 0 replies; 4+ messages in thread
From: Patrick Ohly @ 2015-04-09 9:24 UTC (permalink / raw)
To: openembedded-core
Plain print has two drawbacks:
- the output is handled by different processes, causing the message
about found tests to appear randomly after the result of the initial
tests
- the output is not part of the bitbake console output
Affects image testing with testimage.bbclass.
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
---
meta/lib/oeqa/oetest.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py
index 0b7e7dc..a3c5c1d 100644
--- a/meta/lib/oeqa/oetest.py
+++ b/meta/lib/oeqa/oetest.py
@@ -11,6 +11,7 @@ import os, re, mmap
import unittest
import inspect
import subprocess
+import bb
from oeqa.utils.decorators import LogResults
def loadTests(tc, type="runtime"):
@@ -33,8 +34,8 @@ def loadTests(tc, type="runtime"):
def runTests(tc, type="runtime"):
suite = loadTests(tc, type)
- print("Test modules %s" % tc.testslist)
- print("Found %s tests" % suite.countTestCases())
+ bb.note("Test modules %s" % tc.testslist)
+ bb.note("Found %s tests" % suite.countTestCases())
runner = unittest.TextTestRunner(verbosity=2)
result = runner.run(suite)
--
1.8.4.5
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH 2/3] oeqa/runtime/systemd.py: skip instead of failing without avahi
2015-04-09 9:24 [PATCH 0/3] testimage improvements Patrick Ohly
2015-04-09 9:24 ` [PATCH 1/3] oeqa/oetest.py: use bb logging Patrick Ohly
@ 2015-04-09 9:24 ` Patrick Ohly
2015-04-09 9:24 ` [PATCH 3/3] testimage: sort modules based on dependencies Patrick Ohly
2 siblings, 0 replies; 4+ messages in thread
From: Patrick Ohly @ 2015-04-09 9:24 UTC (permalink / raw)
To: openembedded-core
The SystemdServiceTests assume that avahi-daemon is installed,
which is not necessarily the case depending on the image being
tested.
Better check this dependency before starting the tests and skip
them if the service is not installed. This has to be done for
each test instead of for the entire module, because other
tests in the module can run without avavi.
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
---
meta/lib/oeqa/runtime/systemd.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/meta/lib/oeqa/runtime/systemd.py b/meta/lib/oeqa/runtime/systemd.py
index 5935edd..c74394c 100644
--- a/meta/lib/oeqa/runtime/systemd.py
+++ b/meta/lib/oeqa/runtime/systemd.py
@@ -67,13 +67,19 @@ class SystemdBasicTests(SystemdTest):
class SystemdServiceTests(SystemdTest):
+ def check_for_avahi(self):
+ if not self.hasPackage('avahi-daemon'):
+ raise unittest.SkipTest("Testcase dependency not met: need avahi-daemon installed on target")
+
@skipUnlessPassed('test_systemd_basic')
def test_systemd_status(self):
+ self.check_for_avahi()
self.systemctl('status --full', 'avahi-daemon.service')
@testcase(695)
@skipUnlessPassed('test_systemd_status')
def test_systemd_stop_start(self):
+ self.check_for_avahi()
self.systemctl('stop', 'avahi-daemon.service')
self.systemctl('is-active', 'avahi-daemon.service', expected=3, verbose=True)
self.systemctl('start','avahi-daemon.service')
@@ -82,6 +88,7 @@ class SystemdServiceTests(SystemdTest):
@testcase(696)
@skipUnlessPassed('test_systemd_basic')
def test_systemd_disable_enable(self):
+ self.check_for_avahi()
self.systemctl('disable', 'avahi-daemon.service')
self.systemctl('is-enabled', 'avahi-daemon.service', expected=1)
self.systemctl('enable', 'avahi-daemon.service')
--
1.8.4.5
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH 3/3] testimage: sort modules based on dependencies
2015-04-09 9:24 [PATCH 0/3] testimage improvements Patrick Ohly
2015-04-09 9:24 ` [PATCH 1/3] oeqa/oetest.py: use bb logging Patrick Ohly
2015-04-09 9:24 ` [PATCH 2/3] oeqa/runtime/systemd.py: skip instead of failing without avahi Patrick Ohly
@ 2015-04-09 9:24 ` Patrick Ohly
2 siblings, 0 replies; 4+ messages in thread
From: Patrick Ohly @ 2015-04-09 9:24 UTC (permalink / raw)
To: openembedded-core
TEST_SUITES="auto" is useful to run all suitable tests without
having to hard-code the list. However, it did not take test
dependencies into account, which can be an issue for tests
which really depend on some other test to run first.
To fix this, modules get loaded in the order determined by
TESTS_SUITES, but then get re-ordered based on dependencies
derived from @skipUnlessPassed before running them. The original
order is used to break ties when there are no dependencies, so
reordering only occurs when really necessary.
@skipUnlessPassed gets extended such that it makes the test name
a method depends on available for inspection by the test loader
in oetest.py.
Unfortunately Python's unittest offers no API to inspect tests
in a TestSuite, so the code has to rely on implementation details
to find all tests. The worst that can happen when the implementation
changes is that tests are not found and reordering does not happen.
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
---
meta/classes/testimage.bbclass | 6 ++++-
meta/lib/oeqa/oetest.py | 56 ++++++++++++++++++++++++++++++++++++---
meta/lib/oeqa/utils/decorators.py | 1 +
3 files changed, 59 insertions(+), 4 deletions(-)
diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 6831738..dc163c7 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -16,7 +16,11 @@
# The test names are the module names in meta/lib/oeqa/runtime.
# Each name in TEST_SUITES represents a required test for the image. (no skipping allowed)
# Appending "auto" means that it will try to run all tests that are suitable for the image (each test decides that on it's own).
-# Note that order in TEST_SUITES is important (it's the order tests run) and it influences tests dependencies.
+# Note that order in TEST_SUITES is relevant: tests are run in an order such that
+# tests mentioned in @skipUnlessPassed run before the tests that depend on them,
+# but without such dependencies, tests run in the order in which they are listed
+# in TEST_SUITES.
+#
# A layer can add its own tests in lib/oeqa/runtime, provided it extends BBPATH as normal in its layer.conf.
# TEST_LOG_DIR contains a command ssh log and may contain infromation about what command is running, output and return codes and for qemu a boot log till login.
diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py
index a3c5c1d..da9556a 100644
--- a/meta/lib/oeqa/oetest.py
+++ b/meta/lib/oeqa/oetest.py
@@ -27,9 +27,59 @@ def loadTests(tc, type="runtime"):
setattr(oeTest, "tc", tc)
testloader = unittest.TestLoader()
testloader.sortTestMethodsUsing = None
- suite = testloader.loadTestsFromNames(tc.testslist)
-
- return suite
+ suites = [testloader.loadTestsFromName(name) for name in tc.testslist]
+
+ def getTests(test):
+ '''Return all individual tests executed when running the suite.'''
+ # Unfortunately unittest does not have an API for this, so we have
+ # to rely on implementation details. This only needs to work
+ # for TestSuite containing TestCase.
+ method = getattr(test, '_testMethodName', None)
+ if method:
+ # leaf case: a TestCase
+ yield test
+ else:
+ # Look into TestSuite.
+ tests = getattr(test, '_tests', [])
+ for t1 in tests:
+ for t2 in getTests(t1):
+ yield t2
+
+ # Determine dependencies between suites by looking for @skipUnlessPassed
+ # method annotations. Suite A depends on suite B if any method in A
+ # depends on a method on B.
+ for suite in suites:
+ suite.dependencies = []
+ suite.depth = 0
+ for test in getTests(suite):
+ methodname = getattr(test, '_testMethodName', None)
+ if methodname:
+ method = getattr(test, methodname)
+ depends_on = getattr(method, '_depends_on', None)
+ if depends_on:
+ for dep_suite in suites:
+ if depends_on in [getattr(t, '_testMethodName', None) for t in getTests(dep_suite)]:
+ if dep_suite not in suite.dependencies and \
+ dep_suite is not suite:
+ suite.dependencies.append(dep_suite)
+ break
+ else:
+ bb.warn("Test %s was declared as @skipUnlessPassed('%s') but that test is either not defined or not active. Will run the test anyway." %
+ (test, depends_on))
+ # Use brute-force topological sort to determine ordering. Sort by
+ # depth (higher depth = must run later), with original ordering to
+ # break ties.
+ def set_suite_depth(suite):
+ for dep in suite.dependencies:
+ new_depth = set_suite_depth(dep) + 1
+ if new_depth > suite.depth:
+ suite.depth = new_depth
+ return suite.depth
+ for index, suite in enumerate(suites):
+ set_suite_depth(suite)
+ suite.index = index
+ suites.sort(cmp=lambda a,b: cmp((a.depth, a.index), (b.depth, b.index)))
+ return testloader.suiteClass(suites)
def runTests(tc, type="runtime"):
diff --git a/meta/lib/oeqa/utils/decorators.py b/meta/lib/oeqa/utils/decorators.py
index ff5f278..1e5a890 100644
--- a/meta/lib/oeqa/utils/decorators.py
+++ b/meta/lib/oeqa/utils/decorators.py
@@ -85,6 +85,7 @@ class skipUnlessPassed(object):
raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase)
return f(*args)
wrapped_f.__name__ = f.__name__
+ wrapped_f._depends_on = self.testcase
return wrapped_f
class testcase(object):
--
1.8.4.5
^ permalink raw reply related [flat|nested] 4+ messages in thread